diff --git a/tests/robustness/failpoint/gofail.go b/tests/robustness/failpoint/gofail.go index b6218edb9af..6b8551e28fa 100644 --- a/tests/robustness/failpoint/gofail.go +++ b/tests/robustness/failpoint/gofail.go @@ -59,6 +59,15 @@ var ( BeforeApplyOneConfChangeSleep Failpoint = killAndGofailSleep{"beforeApplyOneConfChange", time.Second} RaftBeforeSaveSleep Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second} RaftAfterSaveSleep Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second} + + // AllowBatchCompactBeforeSetFinishedCompactPanic is used to trigger + // that compactBeforeSetFinishedCompact failpoint only if the current + // revision number is higher than that batch limit. + AllowBatchCompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{ + failpoint: "compactBeforeSetFinishedCompact", + trigger: triggerCompact{multiBatchCompaction: true}, + target: AnyMember, + } ) type goPanicFailpoint struct { diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk index 65c373e897d..bbe8113d091 100644 --- a/tests/robustness/makefile.mk +++ b/tests/robustness/makefile.mk @@ -39,6 +39,11 @@ test-robustness-issue15271: /tmp/etcd-v3.5.7-failpoints/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" +.PHONY: test-robustness-issue17780 +test-robustness-issue17780: /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin + GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17780 --count 100 --failfast --bin-dir=/tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin' make test-robustness && \ + echo "Failed to reproduce" || echo "Successful reproduction" + # Failpoints GOPATH = $(shell go env GOPATH) @@ -84,6 +89,19 @@ $(GOPATH)/bin/gofail: tools/mod/go.mod tools/mod/go.sum make gofail-enable; \ make build; +# REF: https://github.com/etcd-io/etcd/commit/6b034466aa0ac2b46fe01fb5bd2233946f46d453 +/tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin: $(GOPATH)/bin/gofail + rm -rf /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/ + mkdir -p /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/ + cd /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/; \ + git clone --branch release-3.5 https://github.com/etcd-io/etcd.git .; \ + git checkout 6b034466aa0ac2b46fe01fb5bd2233946f46d453; \ + go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ + (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + FAILPOINTS=true ./build; + /tmp/etcd-v3.5.2-failpoints/bin: /tmp/etcd-v3.5.4-failpoints/bin: /tmp/etcd-v3.5.5-failpoints/bin: diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go index e94eab7ae80..cabf4b979fe 100644 --- a/tests/robustness/scenarios.go +++ b/tests/robustness/scenarios.go @@ -199,5 +199,17 @@ func regressionScenarios(t *testing.T) []testScenario { cluster: *e2e.NewConfig(opts...), }) } + scenarios = append(scenarios, testScenario{ + name: "Issue17780", + failpoint: failpoint.AllowBatchCompactBeforeSetFinishedCompactPanic, + profile: traffic.LowTraffic, + traffic: traffic.Issue17780EtcdPutDelete, + cluster: *e2e.NewConfig( + e2e.WithClusterSize(1), + e2e.WithCompactionBatchLimit(100), + e2e.WithSnapshotCount(1000), + e2e.WithGoFailEnabled(true), + ), + }) return scenarios } diff --git a/tests/robustness/traffic/etcd.go b/tests/robustness/traffic/etcd.go index a264341b0d9..34ce04eafb9 100644 --- a/tests/robustness/traffic/etcd.go +++ b/tests/robustness/traffic/etcd.go @@ -63,6 +63,18 @@ var ( {choice: Compact, weight: 5}, }, } + // Issue17780EtcdPutDelete is to create high chance to have more delete + // requests so that it's likely to compact that revision which is tombstone. + Issue17780EtcdPutDelete = etcdTraffic{ + keyCount: 20, + leaseTTL: DefaultLeaseTTL, + largePutSize: 32769, + requests: []choiceWeight[etcdRequestType]{ + {choice: List, weight: 50}, + {choice: Delete, weight: 20}, + {choice: Put, weight: 10}, + }, + } ) type etcdTraffic struct {