From 3ecf29187748b8f83e25d981d0fa5d8d1332833e Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Fri, 4 Nov 2022 15:44:01 +0100 Subject: [PATCH] tests: Add support for lazyfs Signed-off-by: Marek Siarkowicz --- .github/workflows/linearizability.yaml | 3 ++- Makefile | 17 +++++++++++++++++ tests/framework/e2e/etcd_process.go | 3 ++- .../config/lazyfs-member-0.toml | 9 +++++++++ .../config/lazyfs-member-1.toml | 9 +++++++++ .../config/lazyfs-member-2.toml | 9 +++++++++ tests/linearizability/failpoints.go | 19 +++++++++++++++---- tests/linearizability/linearizability_test.go | 1 + 8 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 tests/linearizability/config/lazyfs-member-0.toml create mode 100644 tests/linearizability/config/lazyfs-member-1.toml create mode 100644 tests/linearizability/config/lazyfs-member-2.toml diff --git a/.github/workflows/linearizability.yaml b/.github/workflows/linearizability.yaml index bbf3cae3eda..b12d67531f6 100644 --- a/.github/workflows/linearizability.yaml +++ b/.github/workflows/linearizability.yaml @@ -9,7 +9,8 @@ jobs: with: go-version: "1.19.3" - run: | - pushd tools/mod; go install go.etcd.io/gofail; popd + sudo apt-get -y install cmake libfuse3-dev libfuse3-3 fuse3 + make install-gofail install-lazyfs mkdir -p /tmp/linearizability FAILPOINTS=true make build cat server/etcdserver/raft.fail.go diff --git a/Makefile b/Makefile index 71a8c8e206c..0d80a011771 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,23 @@ verify-proto-annotations: verify-genproto: PASSES="genproto" ./scripts/test.sh +# Tools + +.PHONY: install-gofail +install-gofail: + cd tools/mod; go install go.etcd.io/gofail + +.PHONY: install-lazyfs +install-lazyfs: bin/lazyfs + +bin/lazyfs: + rm /tmp/lazyfs -rf + git clone https://github.com/dsrhaslab/lazyfs /tmp/lazyfs + cd /tmp/lazyfs/libs/libpcache; ./build.sh + cd /tmp/lazyfs/lazyfs; ./build.sh + mkdir -p ./bin + cp /tmp/lazyfs/lazyfs/build/lazyfs ./bin/lazyfs + # Cleanup diff --git a/tests/framework/e2e/etcd_process.go b/tests/framework/e2e/etcd_process.go index 070a77c4d55..9eb0e18a057 100644 --- a/tests/framework/e2e/etcd_process.go +++ b/tests/framework/e2e/etcd_process.go @@ -19,6 +19,7 @@ import ( "fmt" "net/url" "os" + "path/filepath" "strings" "syscall" "testing" @@ -92,7 +93,7 @@ func NewEtcdServerProcess(cfg *EtcdServerProcessConfig) (*EtcdServerProcess, err return nil, fmt.Errorf("could not find etcd binary: %s", cfg.ExecPath) } if !cfg.KeepDataDir { - if err := os.RemoveAll(cfg.DataDirPath); err != nil { + if err := os.RemoveAll(filepath.Join(cfg.DataDirPath, "*")); err != nil { return nil, err } } diff --git a/tests/linearizability/config/lazyfs-member-0.toml b/tests/linearizability/config/lazyfs-member-0.toml new file mode 100644 index 00000000000..b64190ad056 --- /dev/null +++ b/tests/linearizability/config/lazyfs-member-0.toml @@ -0,0 +1,9 @@ +[faults] +fifo_path="/tmp/faults-member-0.fifo" +[cache] +apply_eviction=false +[cache.simple] +custom_size="1gb" +blocks_per_page=1 +[filesystem] +log_all_operations=false diff --git a/tests/linearizability/config/lazyfs-member-1.toml b/tests/linearizability/config/lazyfs-member-1.toml new file mode 100644 index 00000000000..7d5356ac15d --- /dev/null +++ b/tests/linearizability/config/lazyfs-member-1.toml @@ -0,0 +1,9 @@ +[faults] +fifo_path="/tmp/faults-member-1.fifo" +[cache] +apply_eviction=false +[cache.simple] +custom_size="1gb" +blocks_per_page=1 +[filesystem] +log_all_operations=false diff --git a/tests/linearizability/config/lazyfs-member-2.toml b/tests/linearizability/config/lazyfs-member-2.toml new file mode 100644 index 00000000000..05452aeacec --- /dev/null +++ b/tests/linearizability/config/lazyfs-member-2.toml @@ -0,0 +1,9 @@ +[faults] +fifo_path="/tmp/faults-member-2.fifo" +[cache] +apply_eviction=false +[cache.simple] +custom_size="1gb" +blocks_per_page=1 +[filesystem] +log_all_operations=false diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go index 247951c19ef..af9e51e4242 100644 --- a/tests/linearizability/failpoints.go +++ b/tests/linearizability/failpoints.go @@ -21,6 +21,7 @@ import ( "math/rand" "net/http" "net/url" + "os" "strings" "testing" "time" @@ -80,7 +81,8 @@ type Failpoint interface { type killFailpoint struct{} func (f killFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error { - member := clus.Procs[rand.Int()%len(clus.Procs)] + index := rand.Int() % len(clus.Procs) + member := clus.Procs[index] err := member.Kill() if err != nil { return err @@ -89,6 +91,10 @@ func (f killFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.Etcd if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { return err } + err = os.WriteFile(fmt.Sprintf("/tmp/faults-member-%d.fifo", index), []byte("lazyfs::clear-cache\n"), 0666) + if err != nil { + return err + } err = member.Start(ctx) if err != nil { return err @@ -115,15 +121,16 @@ const ( ) func (f goFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error { - var member e2e.EtcdProcess + var index int switch f.target { case AnyMember: - member = clus.Procs[rand.Int()%len(clus.Procs)] + index = rand.Int() % len(clus.Procs) case Leader: - member = clus.Procs[clus.WaitLeader(t)] + index = clus.WaitLeader(t) default: panic("unknown target") } + member := clus.Procs[index] address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort) err := setupGoFailpoint(address, f.failpoint, f.payload) if err != nil { @@ -139,6 +146,10 @@ func (f goFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdPr if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { return err } + err = os.WriteFile(fmt.Sprintf("/tmp/faults-member-%d.fifo", index), []byte("lazyfs::clear-cache\n"), 0666) + if err != nil { + return err + } err = member.Start(ctx) if err != nil { return err diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 9749c5b8a78..fa4b89857cb 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -62,6 +62,7 @@ func TestLinearizability(t *testing.T) { config: *e2e.NewConfig( e2e.WithGoFailEnabled(true), e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints + e2e.WithDataDirPath("/tmp/linearizability"), ), }, {