tests: Add reproduce etcd-io#13766 scenario

serathius · Jan 13, 2023 · 4476f83 · 4476f83
1 parent 6315f1c
commit 4476f83
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 5 deletions.
diff --git a/.github/workflows/linearizability-nightly.yaml b/.github/workflows/linearizability-nightly.yaml
@@ -18,7 +18,7 @@ jobs:
         make build
         mkdir -p /tmp/linearizability
         cat server/etcdserver/raft.fail.go
-        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 500 --failfast --run TestLinearizability --timeout=170m' RESULTS_DIR=/tmp/linearizability make test-linearizability
+        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 400 --failfast --run TestLinearizability --timeout=170m' RESULTS_DIR=/tmp/linearizability make test-linearizability
     - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
       if: always()
       with:

diff --git a/.github/workflows/linearizability.yaml b/.github/workflows/linearizability.yaml
@@ -14,7 +14,7 @@ jobs:
         make build
         mkdir -p /tmp/linearizability
         cat server/etcdserver/raft.fail.go
-        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
+        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 40 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
     - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
       if: always()
       with:

diff --git a/Makefile b/Makefile
@@ -129,6 +129,60 @@ gofail-disable: install-gofail
 install-gofail:
 	cd tools/mod; go install go.etcd.io/gofail@${GOFAIL_VERSION}
 
+# Reproduce historical issues
+
+.PHONY: reproduce-issue14370
+reproduce-issue14370: ./bin/etcd-v3.5.4-failpoints
+	cp ./bin/etcd-v3.5.4-failpoints ./bin/etcd
+	GO_TEST_FLAGS='-v --run=TestLinearizability/Issue14370 --count 100 --failfast' make test-linearizability
+
+.PHONY: reproduce-issue13766
+reproduce-issue13766: ./bin/etcd-v3.5.2-failpoints
+	cp ./bin/etcd-v3.5.2-failpoints ./bin/etcd
+	GO_TEST_FLAGS='-v --run=TestLinearizability/Issue13766 --count 100 --failfast' make test-linearizability
+
+.PHONY: reproduce-issue14685
+reproduce-issue14685: ./bin/etcd-v3.5.5-failpoints
+	cp ./bin/etcd-v3.5.5-failpoints ./bin/etcd
+	GO_TEST_FLAGS='-v --run=TestLinearizability/Issue14685 --count 100 --failfast' make test-linearizability
+
+./bin/etcd-v3.6.0-failpoints:
+	rm -rf /tmp/etcd-release-v3.6.0/
+	mkdir -p /tmp/etcd-release-v3.6.0/
+	cd /tmp/etcd-release-v3.6.0/; \
+	  git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \
+	  make gofail-enable; \
+	  make build;
+	mkdir -p ./bin
+	cp /tmp/etcd-release-v3.6.0/bin/etcd ./bin/etcd-v3.6.0-failpoints
+
+./bin/etcd-v3.5.2-failpoints:
+./bin/etcd-v3.5.4-failpoints:
+./bin/etcd-v3.5.6-failpoints:
+./bin/etcd-v3.5.%-failpoints:
+	rm -rf /tmp/etcd-release-v3.5.$*/
+	mkdir -p /tmp/etcd-release-v3.5.$*/
+	cd /tmp/etcd-release-v3.5.$*/; \
+	  git clone --depth 1 --branch v3.5.$* https://github.com/etcd-io/etcd.git .; \
+	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+	  (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  FAILPOINTS=true ./build;
+	mkdir -p ./bin
+	cp /tmp/etcd-release-v3.5.$*/bin/etcd $@
+
+./bin/etcd-v3.4.23-failpoints:
+./bin/etcd-v3.4.%-failpoints:
+	rm -rf /tmp/etcd-release-v3.4.$*/
+	mkdir -p /tmp/etcd-release-v3.4.$*/
+	cd /tmp/etcd-release-v3.4.$*/; \
+	  git clone --depth 1 --branch v3.4.$* https://github.com/etcd-io/etcd.git .; \
+	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+	  FAILPOINTS=true ./build;
+	mkdir -p ./bin
+	cp /tmp/etcd-release-v3.4.$*/bin/etcd $@
+
 # Cleanup
 
 clean:

diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go
@@ -35,9 +35,9 @@ import (
 
 const (
 	// minimalQPS is used to validate if enough traffic is send to make tests accurate.
-	minimalQPS = 100.0
+	minimalQPS = 200.0
 	// maximalQPS limits number of requests send to etcd to avoid linearizability analysis taking too long.
-	maximalQPS = 200.0
+	maximalQPS = 1000.0
 	// waitBetweenFailpointTriggers
 	waitBetweenFailpointTriggers = time.Second
 )
@@ -54,6 +54,7 @@ func TestLinearizability(t *testing.T) {
 			failpoint: RandomFailpoint,
 			config: *e2e.NewConfig(
 				e2e.WithClusterSize(1),
+				e2e.WithSnapshotCount(100),
 				e2e.WithPeerProxy(true),
 				e2e.WithGoFailEnabled(true),
 				e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
@@ -63,6 +64,7 @@ func TestLinearizability(t *testing.T) {
 			name:      "ClusterOfSize3",
 			failpoint: RandomFailpoint,
 			config: *e2e.NewConfig(
+				e2e.WithSnapshotCount(100),
 				e2e.WithPeerProxy(true),
 				e2e.WithGoFailEnabled(true),
 				e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
@@ -84,6 +86,13 @@ func TestLinearizability(t *testing.T) {
 				e2e.WithGoFailEnabled(true),
 			),
 		},
+		{
+			name:      "Issue13766",
+			failpoint: KillFailpoint,
+			config: *e2e.NewConfig(
+				e2e.WithSnapshotCount(100),
+			),
+		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
@@ -101,7 +110,7 @@ func TestLinearizability(t *testing.T) {
 			}, trafficConfig{
 				minimalQPS:  minimalQPS,
 				maximalQPS:  maximalQPS,
-				clientCount: 8,
+				clientCount: 12,
 				traffic:     DefaultTraffic,
 			})
 			longestHistory, remainingEvents := pickLongestHistory(events)