From b9b336843bcbc3772ef1454e0d33fbf2722657a9 Mon Sep 17 00:00:00 2001
From: lance6716 <lance6716@gmail.com>
Date: Wed, 25 Oct 2023 16:14:03 +0800
Subject: [PATCH] lightning: every HTTP retry should use its own request
 (#47959)

close pingcap/tidb#47930
---
 br/pkg/pdutil/pd.go             | 13 ++++++++-----
 br/pkg/pdutil/pd_serial_test.go | 11 ++++++++++-
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go
index 9d403017c69d6..851009e7aeeb0 100644
--- a/br/pkg/pdutil/pd.go
+++ b/br/pkg/pdutil/pd.go
@@ -164,13 +164,16 @@ func pdRequestWithCode(
 		return 0, nil, errors.Trace(err)
 	}
 	reqURL := fmt.Sprintf("%s/%s", u, prefix)
-	req, err := http.NewRequestWithContext(ctx, method, reqURL, body)
-	if err != nil {
-		return 0, nil, errors.Trace(err)
-	}
-	var resp *http.Response
+	var (
+		req  *http.Request
+		resp *http.Response
+	)
 	count := 0
 	for {
+		req, err = http.NewRequestWithContext(ctx, method, reqURL, body)
+		if err != nil {
+			return 0, nil, errors.Trace(err)
+		}
 		resp, err = cli.Do(req) //nolint:bodyclose
 		count++
 		failpoint.Inject("InjectClosed", func(v failpoint.Value) {
diff --git a/br/pkg/pdutil/pd_serial_test.go b/br/pkg/pdutil/pd_serial_test.go
index 32f69106b2139..329c34c9ed810 100644
--- a/br/pkg/pdutil/pd_serial_test.go
+++ b/br/pkg/pdutil/pd_serial_test.go
@@ -3,6 +3,7 @@
 package pdutil
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"encoding/json"
@@ -186,8 +187,16 @@ func TestPDRequestRetry(t *testing.T) {
 		w.WriteHeader(http.StatusOK)
 	}))
 	cli := http.DefaultClient
+	cli.Transport = http.DefaultTransport.(*http.Transport).Clone()
+	// although the real code doesn't disable keep alive, we need to disable it
+	// in test to avoid the connection being reused and #47930 can't appear. The
+	// real code will only meet #47930 when go's internal http client just dropped
+	// all idle connections.
+	cli.Transport.(*http.Transport).DisableKeepAlives = true
+
 	taddr := ts.URL
-	_, reqErr := pdRequest(ctx, taddr, "", cli, http.MethodGet, nil)
+	body := bytes.NewBuffer([]byte("test"))
+	_, reqErr := pdRequest(ctx, taddr, "", cli, http.MethodPost, body)
 	require.NoError(t, reqErr)
 	ts.Close()
 	count = 0