diff --git a/.github/workflows/linearizability.yaml b/.github/workflows/linearizability.yaml index 89bf14ac275b..d5f437ae12be 100644 --- a/.github/workflows/linearizability.yaml +++ b/.github/workflows/linearizability.yaml @@ -14,7 +14,7 @@ jobs: make build mkdir -p /tmp/linearizability cat server/etcdserver/raft.fail.go - EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability + EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 40 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 if: always() with: diff --git a/Makefile b/Makefile index 1044e776904b..bb4abdfb1aae 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,60 @@ gofail-disable: install-gofail install-gofail: cd tools/mod; go install go.etcd.io/gofail@${GOFAIL_VERSION} +# Reproduce historical issues + +.PHONY: reproduce-issue14370 +reproduce-issue14370: ./bin/etcd-v3.5.4-failpoints + cp ./bin/etcd-v3.5.4-failpoints ./bin/etcd + GO_TEST_FLAGS='-v --run=TestLinearizability/Issue14370 --count 100 --failfast' make test-linearizability + +.PHONY: reproduce-issue13766 +reproduce-issue13766: ./bin/etcd-v3.5.2-failpoints + cp ./bin/etcd-v3.5.2-failpoints ./bin/etcd + GO_TEST_FLAGS='-v --run=TestLinearizability/Issue13766 --count 100 --failfast' make test-linearizability + +.PHONY: reproduce-issue14685 +reproduce-issue14685: ./bin/etcd-v3.5.5-failpoints + cp ./bin/etcd-v3.5.5-failpoints ./bin/etcd + GO_TEST_FLAGS='-v --run=TestLinearizability/Issue14685 --count 100 --failfast' make test-linearizability + +./bin/etcd-v3.6.0-failpoints: + rm -rf /tmp/etcd-release-v3.6.0/ + mkdir -p /tmp/etcd-release-v3.6.0/ + cd /tmp/etcd-release-v3.6.0/; \ + git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \ + make gofail-enable; \ + make build; + mkdir -p ./bin + cp /tmp/etcd-release-v3.6.0/bin/etcd ./bin/etcd-v3.6.0-failpoints + +./bin/etcd-v3.5.2-failpoints: +./bin/etcd-v3.5.4-failpoints: +./bin/etcd-v3.5.6-failpoints: +./bin/etcd-v3.5.%-failpoints: + rm -rf /tmp/etcd-release-v3.5.$*/ + mkdir -p /tmp/etcd-release-v3.5.$*/ + cd /tmp/etcd-release-v3.5.$*/; \ + git clone --depth 1 --branch v3.5.$* https://github.com/etcd-io/etcd.git .; \ + go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ + (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + FAILPOINTS=true ./build; + mkdir -p ./bin + cp /tmp/etcd-release-v3.5.$*/bin/etcd $@ + +./bin/etcd-v3.4.23-failpoints: +./bin/etcd-v3.4.%-failpoints: + rm -rf /tmp/etcd-release-v3.4.$*/ + mkdir -p /tmp/etcd-release-v3.4.$*/ + cd /tmp/etcd-release-v3.4.$*/; \ + git clone --depth 1 --branch v3.4.$* https://github.com/etcd-io/etcd.git .; \ + go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ + FAILPOINTS=true ./build; + mkdir -p ./bin + cp /tmp/etcd-release-v3.4.$*/bin/etcd $@ + # Cleanup clean: diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index bf1e1fef5650..2538cdaf8cb4 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -35,9 +35,9 @@ import ( const ( // minimalQPS is used to validate if enough traffic is send to make tests accurate. - minimalQPS = 100.0 + minimalQPS = 200.0 // maximalQPS limits number of requests send to etcd to avoid linearizability analysis taking too long. - maximalQPS = 200.0 + maximalQPS = 1000.0 // waitBetweenFailpointTriggers waitBetweenFailpointTriggers = time.Second ) @@ -54,6 +54,7 @@ func TestLinearizability(t *testing.T) { failpoint: RandomFailpoint, config: *e2e.NewConfig( e2e.WithClusterSize(1), + e2e.WithSnapshotCount(100), e2e.WithPeerProxy(true), e2e.WithGoFailEnabled(true), e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints @@ -63,6 +64,7 @@ func TestLinearizability(t *testing.T) { name: "ClusterOfSize3", failpoint: RandomFailpoint, config: *e2e.NewConfig( + e2e.WithSnapshotCount(100), e2e.WithPeerProxy(true), e2e.WithGoFailEnabled(true), e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints @@ -84,6 +86,13 @@ func TestLinearizability(t *testing.T) { e2e.WithGoFailEnabled(true), ), }, + { + name: "Issue13766", + failpoint: KillFailpoint, + config: *e2e.NewConfig( + e2e.WithSnapshotCount(100), + ), + }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { @@ -101,7 +110,7 @@ func TestLinearizability(t *testing.T) { }, trafficConfig{ minimalQPS: minimalQPS, maximalQPS: maximalQPS, - clientCount: 8, + clientCount: 12, traffic: DefaultTraffic, }) longestHistory, remainingEvents := pickLongestHistory(events)