From 147844e4cd4160a983bbda7d546e096a5a8353a4 Mon Sep 17 00:00:00 2001
From: Wei Fu <fuweid89@gmail.com>
Date: Thu, 30 May 2024 18:04:29 +0800
Subject: [PATCH] tests: add robustness test for issue 17780

Signed-off-by: Wei Fu <fuweid89@gmail.com>
---
 tests/robustness/failpoint/gofail.go |  9 +++++++++
 tests/robustness/makefile.mk         | 23 +++++++++++++++++++++++
 tests/robustness/scenarios.go        | 19 +++++++++++++++++++
 tests/robustness/traffic/etcd.go     | 12 ++++++++++++
 4 files changed, 63 insertions(+)

diff --git a/tests/robustness/failpoint/gofail.go b/tests/robustness/failpoint/gofail.go
index b6218edb9af9..6b8551e28fa8 100644
--- a/tests/robustness/failpoint/gofail.go
+++ b/tests/robustness/failpoint/gofail.go
@@ -59,6 +59,15 @@ var (
 	BeforeApplyOneConfChangeSleep            Failpoint = killAndGofailSleep{"beforeApplyOneConfChange", time.Second}
 	RaftBeforeSaveSleep                      Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second}
 	RaftAfterSaveSleep                       Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second}
+
+	// AllowBatchCompactBeforeSetFinishedCompactPanic is used to trigger
+	// that compactBeforeSetFinishedCompact failpoint only if the current
+	// revision number is higher than that batch limit.
+	AllowBatchCompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{
+		failpoint: "compactBeforeSetFinishedCompact",
+		trigger:   triggerCompact{multiBatchCompaction: true},
+		target:    AnyMember,
+	}
 )
 
 type goPanicFailpoint struct {
diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk
index 65c373e897da..119bbfdc3db9 100644
--- a/tests/robustness/makefile.mk
+++ b/tests/robustness/makefile.mk
@@ -39,6 +39,11 @@ test-robustness-issue15271: /tmp/etcd-v3.5.7-failpoints/bin
 	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \
 	 echo "Failed to reproduce" || echo "Successful reproduction"
 
+.PHONY: test-robustness-issue17780
+test-robustness-issue17780: /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin
+	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17780 --count 100 --failfast --bin-dir=/tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin' make test-robustness && \
+	  echo "Failed to reproduce" || echo "Successful reproduction"
+
 # Failpoints
 
 GOPATH = $(shell go env GOPATH)
@@ -84,6 +89,24 @@ $(GOPATH)/bin/gofail: tools/mod/go.mod tools/mod/go.sum
 	  make gofail-enable; \
 	  make build;
 
+# NOTE: The commit 6b034466 is to introduce compactBeforeSetFinishedCompact
+# failpoint and only >= v3.5.14 releases contain failpoint. In order to reproduce
+# the issue 17780, we have to clone the whole branch and checkout that commit.
+# So that built etcd binary won't contain fix patch for issue 17780.
+#
+# REF: https://github.com/etcd-io/etcd/commit/6b034466aa0ac2b46fe01fb5bd2233946f46d453
+/tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin: $(GOPATH)/bin/gofail
+	rm -rf /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/
+	mkdir -p /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/
+	cd /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/; \
+		git clone --branch release-3.5 https://github.com/etcd-io/etcd.git .; \
+		git checkout 6b034466aa0ac2b46fe01fb5bd2233946f46d453; \
+		go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+		(cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+		(cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+		(cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+		FAILPOINTS=true ./build;
+
 /tmp/etcd-v3.5.2-failpoints/bin:
 /tmp/etcd-v3.5.4-failpoints/bin:
 /tmp/etcd-v3.5.5-failpoints/bin:
diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go
index cca05dea49c0..095cedbe9a27 100644
--- a/tests/robustness/scenarios.go
+++ b/tests/robustness/scenarios.go
@@ -198,5 +198,24 @@ func regressionScenarios(t *testing.T) []testScenario {
 			cluster:   *e2e.NewConfig(opts...),
 		})
 	}
+	scenarios = append(scenarios, testScenario{
+		name:      "Issue17780",
+		failpoint: failpoint.AllowBatchCompactBeforeSetFinishedCompactPanic,
+		profile: traffic.Profile{
+			Name:        "LowTraffic",
+			MinimalQPS:  50,
+			MaximalQPS:  60,
+			ClientCount: 3,
+			// TODO(fuweid): change it to reasonable value when #17968 has been fixed.
+			MaxNonUniqueRequestConcurrency: 10000,
+		},
+		traffic: traffic.Issue17780EtcdPutDelete,
+		cluster: *e2e.NewConfig(
+			e2e.WithClusterSize(1),
+			e2e.WithCompactionBatchLimit(100),
+			e2e.WithSnapshotCount(1000),
+			e2e.WithGoFailEnabled(true),
+		),
+	})
 	return scenarios
 }
diff --git a/tests/robustness/traffic/etcd.go b/tests/robustness/traffic/etcd.go
index 4e80c633d571..b5d1013f32a9 100644
--- a/tests/robustness/traffic/etcd.go
+++ b/tests/robustness/traffic/etcd.go
@@ -61,6 +61,18 @@ var (
 			{choice: LargePut, weight: 5},
 		},
 	}
+	// Issue17780EtcdPutDelete is to create high chance to have more delete
+	// requests so that it's likely to compact that revision which is tombstone.
+	Issue17780EtcdPutDelete = etcdTraffic{
+		keyCount:     20,
+		leaseTTL:     DefaultLeaseTTL,
+		largePutSize: 32769,
+		requests: []choiceWeight[etcdRequestType]{
+			{choice: List, weight: 50},
+			{choice: Delete, weight: 20},
+			{choice: Put, weight: 10},
+		},
+	}
 )
 
 type etcdTraffic struct {