tests: Add support for lazyfs

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>

.github/workflows/linearizability.yaml

@@ -9,7 +9,8 @@ jobs:
       with:
         go-version: "1.19.3"
     - run: |
-        pushd tools/mod; go install go.etcd.io/gofail; popd
+        sudo apt-get -y install cmake libfuse3-dev libfuse3-3 fuse3
+        make install-gofail install-lazyfs
         mkdir -p /tmp/linearizability
         FAILPOINTS=true make build
         cat server/etcdserver/raft.fail.go

Makefile

@@ -105,6 +105,23 @@ verify-proto-annotations:
 verify-genproto:
 	PASSES="genproto" ./scripts/test.sh
 
+# Tools
+
+.PHONY: install-gofail
+install-gofail:
+	cd tools/mod; go install go.etcd.io/gofail
+
+.PHONY: install-lazyfs
+install-lazyfs: bin/lazyfs
+
+bin/lazyfs:
+	rm /tmp/lazyfs -rf
+	git clone https://github.com/dsrhaslab/lazyfs /tmp/lazyfs
+	cd /tmp/lazyfs/libs/libpcache; ./build.sh
+	cd /tmp/lazyfs/lazyfs; ./build.sh
+	mkdir -p ./bin
+	cp /tmp/lazyfs/lazyfs/build/lazyfs ./bin/lazyfs
+
 
 # Cleanup
 

tests/framework/e2e/etcd_process.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"testing"
@@ -92,7 +93,7 @@ func NewEtcdServerProcess(cfg *EtcdServerProcessConfig) (*EtcdServerProcess, err
 		return nil, fmt.Errorf("could not find etcd binary: %s", cfg.ExecPath)
 	}
 	if !cfg.KeepDataDir {
-		if err := os.RemoveAll(cfg.DataDirPath); err != nil {
+		if err := os.RemoveAll(filepath.Join(cfg.DataDirPath, "*")); err != nil {
 			return nil, err
 		}
 	}

tests/linearizability/config/lazyfs-member-0.toml

@@ -0,0 +1,9 @@
+[faults]
+fifo_path="/tmp/faults-member-0.fifo"
+[cache]
+apply_eviction=false
+[cache.simple]
+custom_size="1gb"
+blocks_per_page=1
+[filesystem]
+log_all_operations=false

tests/linearizability/config/lazyfs-member-1.toml

@@ -0,0 +1,9 @@
+[faults]
+fifo_path="/tmp/faults-member-1.fifo"
+[cache]
+apply_eviction=false
+[cache.simple]
+custom_size="1gb"
+blocks_per_page=1
+[filesystem]
+log_all_operations=false

tests/linearizability/config/lazyfs-member-2.toml

@@ -0,0 +1,9 @@
+[faults]
+fifo_path="/tmp/faults-member-2.fifo"
+[cache]
+apply_eviction=false
+[cache.simple]
+custom_size="1gb"
+blocks_per_page=1
+[filesystem]
+log_all_operations=false

tests/linearizability/failpoints.go

@@ -21,6 +21,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -80,7 +81,8 @@ type Failpoint interface {
 type killFailpoint struct{}
 
 func (f killFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error {
-	member := clus.Procs[rand.Int()%len(clus.Procs)]
+	index := rand.Int() % len(clus.Procs)
+	member := clus.Procs[index]
 	err := member.Kill()
 	if err != nil {
 		return err
@@ -89,6 +91,10 @@ func (f killFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.Etcd
 	if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
 		return err
 	}
+	err = os.WriteFile(fmt.Sprintf("/tmp/faults-member-%d.fifo", index), []byte("lazyfs::clear-cache\n"), 0666)
+	if err != nil {
+		return err
+	}
 	err = member.Start(ctx)
 	if err != nil {
 		return err
@@ -115,15 +121,16 @@ const (
 )
 
 func (f goFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error {
-	var member e2e.EtcdProcess
+	var index int
 	switch f.target {
 	case AnyMember:
-		member = clus.Procs[rand.Int()%len(clus.Procs)]
+		index = rand.Int() % len(clus.Procs)
 	case Leader:
-		member = clus.Procs[clus.WaitLeader(t)]
+		index = clus.WaitLeader(t)
 	default:
 		panic("unknown target")
 	}
+	member := clus.Procs[index]
 	address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
 	err := setupGoFailpoint(address, f.failpoint, f.payload)
 	if err != nil {
@@ -139,6 +146,10 @@ func (f goFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdPr
 	if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
 		return err
 	}
+	err = os.WriteFile(fmt.Sprintf("/tmp/faults-member-%d.fifo", index), []byte("lazyfs::clear-cache\n"), 0666)
+	if err != nil {
+		return err
+	}
 	err = member.Start(ctx)
 	if err != nil {
 		return err

tests/linearizability/linearizability_test.go

@@ -62,6 +62,7 @@ func TestLinearizability(t *testing.T) {
 			config: *e2e.NewConfig(
 				e2e.WithGoFailEnabled(true),
 				e2e.WithCompactionBatchLimit(100), // required for compactBeforeCommitBatch and compactAfterCommitBatch failpoints
+				e2e.WithDataDirPath("/tmp/linearizability"),
 			),
 		},
 		{