*: replace gofail with new failpoint implementation (pingcap#165)

overvenus · Apr 17, 2019 · ea3b27e · ea3b27e
1 parent fcbafa0
commit ea3b27e
Show file tree

Hide file tree

Showing 11 changed files with 72 additions and 143 deletions.
diff --git a/Makefile b/Makefile
@@ -23,8 +23,8 @@ LINUX     := "Linux"
 MAC       := "Darwin"
 PACKAGES  := $$(go list ./...| grep -vE 'vendor|cmd|test|proto|diff|bin')
 
-GOFAIL_ENABLE  := $$(find $$PWD/lightning/ -type d | xargs gofail enable)
-GOFAIL_DISABLE := $$(find $$PWD/lightning/ -type d | xargs gofail disable)
+FAILPOINT_ENABLE  := $$(failpoint-ctl enable $$PWD/lightning/)
+FAILPOINT_DISABLE := $$(failpoint-ctl disable $$PWD/lightning/)
 
 RACE_FLAG =
 ifeq ("$(WITH_RACE)", "1")
@@ -64,26 +64,22 @@ lightning-ctl:
 
 test:
 	mkdir -p "$(TEST_DIR)"
-	@hash gofail || $(GO) get -v github.com/pingcap/gofail
-	# ^FIXME switch back to etcd-io/gofail after their issue #16 is fixed.
-	$(GOFAIL_ENABLE)
+	$(FAILPOINT_ENABLE)
 	@export log_level=error;\
-	$(GOTEST) -cover -covermode=count -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES)
-	$(GOFAIL_DISABLE)
+	$(GOTEST) -cover -covermode=count -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES) || ( $(FAILPOINT_DISABLE) && exit 1 )
+	$(FAILPOINT_DISABLE)
 
 lightning_for_integration_test:
-	@hash gofail || $(GO) get -v github.com/pingcap/gofail
-	# ^FIXME switch back to etcd-io/gofail after their issue #16 is fixed.
-	$(GOFAIL_ENABLE)
+	$(FAILPOINT_ENABLE)
 	$(GOTEST) -c -cover -covermode=count \
 		-coverpkg=github.com/pingcap/tidb-lightning/... \
 		-o $(LIGHTNING_BIN).test \
-		github.com/pingcap/tidb-lightning/cmd/tidb-lightning
+		github.com/pingcap/tidb-lightning/cmd/tidb-lightning || ( $(FAILPOINT_DISABLE) && exit 1 )
 	$(GOTEST) -c -cover -covermode=count \
 		-coverpkg=github.com/pingcap/tidb-lightning/... \
 		-o $(LIGHTNING_CTL_BIN).test \
-		github.com/pingcap/tidb-lightning/cmd/tidb-lightning-ctl
-	$(GOFAIL_DISABLE)
+		github.com/pingcap/tidb-lightning/cmd/tidb-lightning-ctl || ( $(FAILPOINT_DISABLE) && exit 1 )
+	$(FAILPOINT_DISABLE)
 
 integration_test: lightning_for_integration_test
 	@which bin/tidb-server
@@ -94,7 +90,7 @@ integration_test: lightning_for_integration_test
 
 coverage:
 	GO111MODULE=off go get github.com/wadey/gocovmerge
-	gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go" > "$(TEST_DIR)/all_cov.out"
+	gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
 ifeq ("$(JenkinsCI)", "1")
 	GO111MODULE=off go get github.com/mattn/goveralls
 	@goveralls -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
@@ -107,8 +103,11 @@ update:
 	GO111MODULE=on go mod verify
 	GO111MODULE=on go mod tidy
 
-gofail-enable:
-	$(GOFAIL_ENABLE)
+install-failpoint:
+	@hash failpoint-ctl || $(GO) get -v github.com/pingcap/failpoint/failpoint-ctl
 
-gofail-disable:
-	$(GOFAIL_DISABLE)
+failpoint-enable: install-failpoint
+	$(FAILPOINT_ENABLE)
+
+failpoint-disable: install-failpoint
+	$(FAILPOINT_DISABLE)
diff --git a/cmd/tidb-lightning-ctl/main_test.go b/cmd/tidb-lightning-ctl/main_test.go
@@ -17,9 +17,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-
-	// ensure vendor contains it.
-	_ "github.com/pingcap/gofail/runtime"
 )
 
 func TestRunMain(t *testing.T) {

diff --git a/cmd/tidb-lightning/main_test.go b/cmd/tidb-lightning/main_test.go
@@ -19,9 +19,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-
-	// ensure vendor contains it.
-	_ "github.com/pingcap/gofail/runtime"
 )
 
 func TestRunMain(t *testing.T) {

diff --git a/go.mod b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/montanaflynn/stats v0.5.0 // indirect
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
 	github.com/pingcap/errors v0.11.1
-	github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3
+	github.com/pingcap/failpoint v0.0.0-20190416093020-cb5345727375
 	github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562
 	github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596
 	github.com/pingcap/parser v0.0.0-20190325012055-cc0fa08f99ca

diff --git a/go.sum b/go.sum
diff --git a/lightning/kv/importer.go b/lightning/kv/importer.go
@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/satori/go.uuid"
 	"google.golang.org/grpc"
 
@@ -182,18 +183,14 @@ func (importer *Importer) OpenEngine(
 	openCounter.Inc()
 	common.AppLogger.Infof("[%s] open engine %s", tag, engineUUID)
 
-	// gofail: var FailIfEngineCountExceeds int
-	// {
-	// 	closedCounter := metric.ImporterEngineCounter.WithLabelValues("closed")
-	// 	openCount := metric.ReadCounter(openCounter)
-	// 	closedCount := metric.ReadCounter(closedCounter)
-	// 	if openCount - closedCount > float64(FailIfEngineCountExceeds) {
-	// 		panic(fmt.Sprintf("forcing failure due to FailIfEngineCountExceeds: %v - %v >= %d", openCount, closedCount, FailIfEngineCountExceeds))
-	// 	}
-	// }
-	// goto RETURN
-
-	// gofail: RETURN:
+	failpoint.Inject("FailIfEngineCountExceeds", func(val failpoint.Value) {
+		closedCounter := metric.ImporterEngineCounter.WithLabelValues("closed")
+		openCount := metric.ReadCounter(openCounter)
+		closedCount := metric.ReadCounter(closedCounter)
+		if injectValue := val.(int); openCount-closedCount > float64(injectValue) {
+			panic(fmt.Sprintf("forcing failure due to FailIfEngineCountExceeds: %v - %v >= %d", openCount, closedCount, injectValue))
+		}
+	})
 
 	return &OpenedEngine{
 		importer: importer,

diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go
@@ -30,6 +30,7 @@ import (
 	"github.com/coreos/go-semver/semver"
 	"github.com/cznic/mathutil"
 	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	sstpb "github.com/pingcap/kvproto/pkg/import_sstpb"
 	"github.com/pingcap/parser/model"
 	tidbcfg "github.com/pingcap/tidb/config"
@@ -357,43 +358,37 @@ func (rc *RestoreController) listenCheckpointUpdates() {
 
 		lock.Unlock()
 
-		// gofail: var FailIfImportedChunk int
-		// if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(FailIfImportedChunk) {
-		// 	rc.checkpointsWg.Done()
-		// 	rc.checkpointsWg.Wait()
-		// 	panic("forcing failure due to FailIfImportedChunk")
-		// }
-		// goto RETURN1
-
-		// gofail: RETURN1:
-
-		// gofail: var FailIfStatusBecomes int
-		// if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID >= 0 && int(merger.Status) == FailIfStatusBecomes {
-		// 	rc.checkpointsWg.Done()
-		// 	rc.checkpointsWg.Wait()
-		// 	panic("forcing failure due to FailIfStatusBecomes")
-		// }
-		// goto RETURN2
-
-		// gofail: RETURN2:
-
-		// gofail: var FailIfIndexEngineImported int
-		// if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID == wholeTableEngineID && merger.Status == CheckpointStatusIndexImported && FailIfIndexEngineImported > 0 {
-		// 	rc.checkpointsWg.Done()
-		// 	rc.checkpointsWg.Wait()
-		// 	panic("forcing failure due to FailIfIndexEngineImported")
-		// }
-		// goto RETURN3
+		failpoint.Inject("FailIfImportedChunk", func(val failpoint.Value) {
+			if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(val.(int)) {
+				rc.checkpointsWg.Done()
+				rc.checkpointsWg.Wait()
+				panic("forcing failure due to FailIfImportedChunk")
+			}
+		})
 
-		// gofail: RETURN3:
+		failpoint.Inject("FailIfStatusBecomes", func(val failpoint.Value) {
+			if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID >= 0 && int(merger.Status) == val.(int) {
+				rc.checkpointsWg.Done()
+				rc.checkpointsWg.Wait()
+				panic("forcing failure due to FailIfStatusBecomes")
+			}
+		})
 
-		// gofail: var KillIfImportedChunk int
-		// if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(KillIfImportedChunk) {
-		// 	common.KillMySelf()
-		// }
-		// goto RETURN4
+		failpoint.Inject("FailIfIndexEngineImported", func(val failpoint.Value) {
+			if merger, ok := scp.merger.(*StatusCheckpointMerger); ok &&
+				merger.EngineID == wholeTableEngineID &&
+				merger.Status == CheckpointStatusIndexImported && val.(int) > 0 {
+				rc.checkpointsWg.Done()
+				rc.checkpointsWg.Wait()
+				panic("forcing failure due to FailIfIndexEngineImported")
+			}
+		})
 
-		// gofail: RETURN4:
+		failpoint.Inject("KillIfImportedChunk", func(val failpoint.Value) {
+			if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(val.(int)) {
+				common.KillMySelf()
+			}
+		})
 	}
 	rc.checkpointsWg.Done()
 }
@@ -671,8 +666,9 @@ func (t *TableRestore) restoreEngines(ctx context.Context, rc *RestoreController
 			rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusImported)
 		}
 
-		// gofail: var FailBeforeIndexEngineImported struct{}
-		//  panic("forcing failure due to FailBeforeIndexEngineImported")
+		failpoint.Inject("FailBeforeIndexEngineImported", func() {
+			panic("forcing failure due to FailBeforeIndexEngineImported")
+		})
 
 		rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, err, CheckpointStatusIndexImported)
 		if err != nil {
@@ -1239,7 +1235,7 @@ func (tr *TableRestore) importKV(ctx context.Context, closedEngine *kv.ClosedEng
 	metric.ImportSecondsHistogram.Observe(dur.Seconds())
 	common.AppLogger.Infof("[%s] kv deliver all flushed, takes %v", closedEngine.Tag(), dur)
 
-	// gofail: var SlowDownImport struct{}
+	failpoint.Inject("SlowDownImport", func() {})
 
 	return nil
 }

diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh
@@ -57,7 +57,7 @@ PARTIAL_IMPORT_QUERY="$PARTIAL_IMPORT_QUERY AS s;"
 
 # Set the failpoint to kill the lightning instance as soon as one table is imported
 # If checkpoint does work, this should only kill 9 instances of lightnings.
-export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return'
+export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return'
 
 # Start importing the tables.
 run_sql 'DROP DATABASE IF EXISTS cppk_tsr'
@@ -72,7 +72,7 @@ for i in $(seq "$TABLE_COUNT"); do
 done
 set -e
 
-export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500)'
+export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500)'
 set +e
 for i in $(seq "$TABLE_COUNT"); do
     echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********"
@@ -84,7 +84,7 @@ set -e
 run_sql 'DROP DATABASE IF EXISTS cppk_tsr'
 run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk'
 
-export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)'
+export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)'
 
 set +e
 for i in $(seq "$TABLE_COUNT"); do

diff --git a/tests/checkpoint_chunks/run.sh b/tests/checkpoint_chunks/run.sh
@@ -46,7 +46,7 @@ done
 # Set the failpoint to kill the lightning instance as soon as
 # one file (after writing totally $ROW_COUNT rows) is imported.
 # If checkpoint does work, this should kill exactly $CHUNK_COUNT instances of lightnings.
-export GOFAIL_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)"
+export GO_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)"
 
 # Start importing the tables.
 run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
@@ -68,7 +68,7 @@ run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cpch'
 
 # Set the failpoint to kill the lightning instance as soon as one chunk is imported, via signal mechanism
 # If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
-export GOFAIL_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/KillIfImportedChunk=return($ROW_COUNT)"
+export GO_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/KillIfImportedChunk=return($ROW_COUNT)"
 
 for i in $(seq "$CHUNK_COUNT"); do
     echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********"
@@ -102,7 +102,7 @@ rm -f "$TEST_DIR/cpch.pb"
 
 # Set the failpoint to kill the lightning instance as soon as one chunk is imported
 # If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
-export GOFAIL_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)"
+export GO_FAILPOINTS="github.com/pingcap/tidb-lightning/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)"
 set +e
 for i in $(seq "$CHUNK_COUNT"); do
     echo "******** Importing Chunk using File checkpoint Now (step $i/$CHUNK_COUNT) ********"

diff --git a/tests/checkpoint_engines/run.sh b/tests/checkpoint_engines/run.sh
@@ -46,7 +46,7 @@ run_sql 'DROP DATABASE cpeng;'
 rm -f "/tmp/tidb_lightning_checkpoint.pb"
 
 # Data engine part
-export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(140)'
+export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(140)'
 set +e
 for i in $(seq "$ENGINE_COUNT"); do
     echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********"

diff --git a/tests/restore/run.sh b/tests/restore/run.sh
@@ -28,7 +28,7 @@ done
 
 # Count OpenEngine and CloseEngine events.
 # Abort if number of unbalanced OpenEngine is >= 4
-export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/kv/FailIfEngineCountExceeds=return(4)'
+export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/kv/FailIfEngineCountExceeds=return(4)'
 
 # Start importing
 run_sql 'DROP DATABASE IF EXISTS restore_tsr'