diff --git a/cdc/Makefile b/cdc/Makefile index 8e9b1249..85c18258 100644 --- a/cdc/Makefile +++ b/cdc/Makefile @@ -8,7 +8,7 @@ FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }' CURDIR := $(shell pwd) path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH))) -export PATH := $(CURDIR)/bin:$(path_to_add):$(PATH) +export PATH := $(CURDIR)/bin:$(CURDIR)/tools/bin:$(path_to_add):$(PATH) SHELL := /usr/bin/env bash @@ -73,6 +73,8 @@ LDFLAGS += -X "$(CDC_PKG)/pkg/version.GoVersion=$(GOVERSION)" SCVERSION := stable +include tools/Makefile + default: build buildsucc buildsucc: @@ -97,7 +99,7 @@ build-failpoint: check_failpoint_ctl $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tikv-cdc ./cmd/cdc/main.go $(FAILPOINT_DISABLE) -cdc: +cdc: go-generate $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tikv-cdc ./cmd/cdc/main.go debug: @@ -109,7 +111,7 @@ kafka_consumer: install: go install ./... -unit_test: check_failpoint_ctl +unit_test: check_failpoint_ctl go-generate mkdir -p "$(TEST_DIR)" $(FAILPOINT_ENABLE) @export log_level=error;\ @@ -117,7 +119,7 @@ unit_test: check_failpoint_ctl || { $(FAILPOINT_DISABLE); exit 1; } $(FAILPOINT_DISABLE) -unit_test_in_verify_ci: check_failpoint_ctl gotestsum gocov gocov-xml +unit_test_in_verify_ci: check_failpoint_ctl go-generate gotestsum gocov gocov-xml mkdir -p "$(TEST_DIR)" $(FAILPOINT_ENABLE) @export log_level=error;\ @@ -171,7 +173,7 @@ check-merge-conflicts: check-gosec: tools/bin/gosec tools/bin/gosec -fmt=junit-xml -out=results.xml -stdout -verbose=text -exclude-dir tests/.unmodify -exclude=G103,G104,G204,G304,G307,G401,G404,G501,G505,G601 ./... -check: check-copyright fmt check-static tidy errdoc check-leaktest-added check-merge-conflicts +check: check-copyright go-generate fmt check-static tidy errdoc check-leaktest-added check-merge-conflicts unit_test_coverage: grep -vE ".*.pb.go|$(CDC_PKG)/testing_utils/.*|$(CDC_PKG)/cdc/kv/testing.go|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out" @@ -192,54 +194,6 @@ install_test_python_dep: @echo "install python requirments for test" pip install --user -q -r ./dm/tests/requirements.txt -tools/bin/failpoint-ctl: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl - -tools/bin/gocovmerge: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/gocovmerge github.com/zhouqiang-cl/gocovmerge - -tools/bin/goveralls: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/goveralls github.com/mattn/goveralls - -tools/bin/golangci-lint: tools/check/go.mod - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b tools/bin v1.55.2 - -tools/bin/mockgen: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/mockgen github.com/golang/mock/mockgen - -tools/bin/protoc-gen-gogofaster: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/protoc-gen-gogofaster github.com/gogo/protobuf/protoc-gen-gogofaster - -tools/bin/protoc-gen-grpc-gateway: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway - -tools/bin/statik: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/statik github.com/rakyll/statik - -gofumpt: - $(GO) install mvdan.cc/gofumpt@latest - -shfmt: - $(GO) install mvdan.cc/sh/v3/cmd/shfmt@latest - -tools/bin/oapi-codegen: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/oapi-codegen github.com/deepmap/oapi-codegen/cmd/oapi-codegen - -gocov: - $(GO) install github.com/axw/gocov/gocov@latest - -gocov-xml: - $(GO) install github.com/AlekSi/gocov-xml@latest - -gotestsum: - $(GO) install gotest.tools/gotestsum@latest - -tools/bin/errdoc-gen: tools/check/go.mod - cd tools/check && $(GO) build -mod=mod -o ../bin/errdoc-gen github.com/pingcap/errors/errdoc-gen - -tools/bin/gosec: - curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b tools/bin v2.18.2 - check_failpoint_ctl: tools/bin/failpoint-ctl failpoint-enable: check_failpoint_ctl @@ -299,11 +253,7 @@ check_third_party_binary: @which scripts/bin/go-ycsb @which scripts/bin/etcdctl -shellcheck: tools/bin/shellcheck - tools/bin/shellcheck ./**/*.sh - -tools/bin/shellcheck: - wget -qO- "https://github.com/koalaman/shellcheck/releases/download/$(SCVERSION)/shellcheck-$(SCVERSION).$(ARCH).x86_64.tar.xz" | tar -xJv - mv "shellcheck-$(SCVERSION)/shellcheck" tools/bin/ - chmod +x tools/bin/shellcheck - rm -rf "shellcheck-$(SCVERSION)" +go-generate: ## Run go generate on all packages. +go-generate: tools/bin/msgp + @echo "go generate" + @go generate ./... diff --git a/cdc/cdc/model/kv.go b/cdc/cdc/model/kv.go index 47162bb1..4497bbbc 100644 --- a/cdc/cdc/model/kv.go +++ b/cdc/cdc/model/kv.go @@ -83,7 +83,7 @@ type RawKVEntry struct { StartTs uint64 `msg:"start_ts"` // Commit or resolved TS CRTs uint64 `msg:"crts"` - ExpiredTs uint64 `msh:"expired_ts"` + ExpiredTs uint64 `msg:"expired_ts"` // Additional debug info RegionID uint64 `msg:"region_id"` diff --git a/cdc/cdc/model/kv_gen.go b/cdc/cdc/model/kv_gen.go index 80b30583..ec5b460d 100644 --- a/cdc/cdc/model/kv_gen.go +++ b/cdc/cdc/model/kv_gen.go @@ -3,37 +3,9 @@ package model // Code generated by github.com/tinylib/msgp DO NOT EDIT. import ( - "reflect" - "github.com/tinylib/msgp/msgp" ) -var ( - fieldNum = getStructFieldNum(RawKVEntry{}) - fieldNameOpType = generateFeildName("op_type") - fieldNameKey = generateFeildName("key") - fieldNameValue = generateFeildName("value") - fieldNameOldValue = generateFeildName("old_value") - fieldNameStartTs = generateFeildName("start_ts") - fieldNameCRTs = generateFeildName("crts") - fieldNameExpiredTs = generateFeildName("expired_ts") - fieldNameRegionID = generateFeildName("region_id") - fieldNameKeySpanID = generateFeildName("keyspan_id") - fieldNameSequence = generateFeildName("sequence") - fieldNames = [][]byte{ - fieldNameOpType, - fieldNameKey, - fieldNameValue, - fieldNameOldValue, - fieldNameStartTs, - fieldNameCRTs, - fieldNameExpiredTs, - fieldNameRegionID, - fieldNameKeySpanID, - fieldNameSequence, - } -) - // DecodeMsg implements msgp.Decodable func (z *OpType) DecodeMsg(dc *msgp.Reader) (err error) { { @@ -182,12 +154,8 @@ func (z *RawKVEntry) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { // map header, size 10 - err = en.Append(0x80 + byte(fieldNum)) - if err != nil { - return - } // write "op_type" - err = en.Append(fieldNameOpType...) + err = en.Append(0x8a, 0xa7, 0x6f, 0x70, 0x5f, 0x74, 0x79, 0x70, 0x65) if err != nil { return } @@ -197,7 +165,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "key" - err = en.Append(fieldNameKey...) + err = en.Append(0xa3, 0x6b, 0x65, 0x79) if err != nil { return } @@ -207,7 +175,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "value" - err = en.Append(fieldNameValue...) + err = en.Append(0xa5, 0x76, 0x61, 0x6c, 0x75, 0x65) if err != nil { return } @@ -217,7 +185,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "old_value" - err = en.Append(fieldNameOldValue...) + err = en.Append(0xa9, 0x6f, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65) if err != nil { return } @@ -227,7 +195,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "start_ts" - err = en.Append(fieldNameStartTs...) + err = en.Append(0xa8, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x73) if err != nil { return } @@ -237,7 +205,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "crts" - err = en.Append(fieldNameCRTs...) + err = en.Append(0xa4, 0x63, 0x72, 0x74, 0x73) if err != nil { return } @@ -247,7 +215,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "expired_ts" - err = en.Append(fieldNameExpiredTs...) + err = en.Append(0xaa, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x74, 0x73) if err != nil { return } @@ -257,7 +225,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "region_id" - err = en.Append(fieldNameRegionID...) + err = en.Append(0xa9, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64) if err != nil { return } @@ -267,7 +235,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "keyspan_id" - err = en.Append(fieldNameKeySpanID...) + err = en.Append(0xaa, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) if err != nil { return } @@ -277,7 +245,7 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { return } // write "sequence" - err = en.Append(fieldNameSequence...) + err = en.Append(0xa8, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65) if err != nil { return } @@ -293,36 +261,35 @@ func (z *RawKVEntry) EncodeMsg(en *msgp.Writer) (err error) { func (z *RawKVEntry) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // map header, size 10 - o = append(o, 0x80+byte(fieldNum)) // string "op_type" - o = append(o, fieldNameOpType...) + o = append(o, 0x8a, 0xa7, 0x6f, 0x70, 0x5f, 0x74, 0x79, 0x70, 0x65) o = msgp.AppendInt(o, int(z.OpType)) // string "key" - o = append(o, fieldNameKey...) + o = append(o, 0xa3, 0x6b, 0x65, 0x79) o = msgp.AppendBytes(o, z.Key) // string "value" - o = append(o, fieldNameValue...) + o = append(o, 0xa5, 0x76, 0x61, 0x6c, 0x75, 0x65) o = msgp.AppendBytes(o, z.Value) // string "old_value" - o = append(o, fieldNameOldValue...) + o = append(o, 0xa9, 0x6f, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65) o = msgp.AppendBytes(o, z.OldValue) // string "start_ts" - o = append(o, fieldNameStartTs...) + o = append(o, 0xa8, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x73) o = msgp.AppendUint64(o, z.StartTs) // string "crts" - o = append(o, fieldNameCRTs...) + o = append(o, 0xa4, 0x63, 0x72, 0x74, 0x73) o = msgp.AppendUint64(o, z.CRTs) // string "expired_ts" - o = append(o, fieldNameExpiredTs...) + o = append(o, 0xaa, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x74, 0x73) o = msgp.AppendUint64(o, z.ExpiredTs) // string "region_id" - o = append(o, fieldNameRegionID...) + o = append(o, 0xa9, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64) o = msgp.AppendUint64(o, z.RegionID) // string "keyspan_id" - o = append(o, fieldNameKeySpanID...) + o = append(o, 0xaa, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) o = msgp.AppendUint64(o, z.KeySpanID) // string "sequence" - o = append(o, fieldNameSequence...) + o = append(o, 0xa8, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65) o = msgp.AppendUint64(o, z.Sequence) return } @@ -423,35 +390,6 @@ func (z *RawKVEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *RawKVEntry) Msgsize() (s int) { - s = 1 - for _, fieldName := range fieldNames { - s += len(fieldName) - } - s += msgp.IntSize + // OpType - msgp.BytesPrefixSize + len(z.Key) + // Key - msgp.BytesPrefixSize + len(z.Value) + // Value - msgp.BytesPrefixSize + len(z.OldValue) + // OldVale - msgp.Uint64Size + // StartTs - msgp.Uint64Size + // CRTs - msgp.Uint64Size + // ExpiredTs - msgp.Uint64Size + // RegionID - msgp.Uint64Size + // KeySpanID - msgp.Uint64Size // Sequence - return -} - -func generateFeildName(feildName string) (bytes []byte) { - l := len(feildName) - bytes = make([]byte, 0, l+1) - bytes = append(bytes, 0xa0+byte(l)) - bytes = append(bytes, []byte(feildName)...) + s = 1 + 8 + msgp.IntSize + 4 + msgp.BytesPrefixSize + len(z.Key) + 6 + msgp.BytesPrefixSize + len(z.Value) + 10 + msgp.BytesPrefixSize + len(z.OldValue) + 9 + msgp.Uint64Size + 5 + msgp.Uint64Size + 11 + msgp.Uint64Size + 10 + msgp.Uint64Size + 11 + msgp.Uint64Size + 9 + msgp.Uint64Size return } - -func getStructFieldNum(structName interface{}) int { - t := reflect.TypeOf(structName) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - return t.NumField() -} diff --git a/cdc/cdc/model/kv_gen_test.go b/cdc/cdc/model/kv_gen_test.go index d8e6d218..dbf9aefd 100644 --- a/cdc/cdc/model/kv_gen_test.go +++ b/cdc/cdc/model/kv_gen_test.go @@ -6,41 +6,11 @@ import ( "bytes" "testing" - "github.com/stretchr/testify/require" "github.com/tinylib/msgp/msgp" ) -func TestGetStructFieldNum(t *testing.T) { - type StructA struct { - field1 int - field2 uint64 - field3 string - } - - require.Equal(t, 3, getStructFieldNum(StructA{})) -} - -func TestGenerateFieldName(t *testing.T) { - s := "keyspan_id" - tbytes := []byte{0xaa, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64} - bytes := generateFeildName(s) - require.Equal(t, bytes, tbytes) -} - func TestMarshalUnmarshalRawKVEntry(t *testing.T) { - v := RawKVEntry{ - OpType: OpTypePut, - Key: []byte("key"), - Value: []byte("value"), - OldValue: []byte("old_value"), - StartTs: 0, - CRTs: 1, - ExpiredTs: 2, - RegionID: 3, - KeySpanID: 4, - Sequence: 5, - } - + v := RawKVEntry{} bts, err := v.MarshalMsg(nil) if err != nil { t.Fatal(err) @@ -52,18 +22,6 @@ func TestMarshalUnmarshalRawKVEntry(t *testing.T) { if len(left) > 0 { t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) } - require.Equal(t, v, RawKVEntry{ - OpType: OpTypePut, - Key: []byte("key"), - Value: []byte("value"), - OldValue: []byte("old_value"), - StartTs: 0, - CRTs: 1, - ExpiredTs: 2, - RegionID: 3, - KeySpanID: 4, - Sequence: 5, - }) left, err = msgp.Skip(bts) if err != nil { @@ -110,19 +68,7 @@ func BenchmarkUnmarshalRawKVEntry(b *testing.B) { } func TestEncodeDecodeRawKVEntry(t *testing.T) { - v := RawKVEntry{ - OpType: OpTypePut, - Key: []byte("key"), - Value: []byte("value"), - OldValue: []byte("old_value"), - StartTs: 0, - CRTs: 1, - ExpiredTs: 2, - RegionID: 3, - KeySpanID: 4, - Sequence: 5, - } - + v := RawKVEntry{} var buf bytes.Buffer msgp.Encode(&buf, &v) @@ -137,8 +83,6 @@ func TestEncodeDecodeRawKVEntry(t *testing.T) { t.Error(err) } - require.Equal(t, v, vn) - buf.Reset() msgp.Encode(&buf, &v) err = msgp.NewReader(&buf).Skip() diff --git a/cdc/cdc/model/kv_test.go b/cdc/cdc/model/kv_test.go index 88f5d3d3..10b17caf 100644 --- a/cdc/cdc/model/kv_test.go +++ b/cdc/cdc/model/kv_test.go @@ -14,10 +14,12 @@ package model import ( + "bytes" "testing" "github.com/stretchr/testify/require" "github.com/tikv/migration/cdc/pkg/regionspan" + "github.com/tinylib/msgp/msgp" ) func TestRegionFeedEvent(t *testing.T) { @@ -58,3 +60,59 @@ func TestRawKVEntry(t *testing.T) { require.Equal(t, "OpType: 1, Key: 123, Value: 345, StartTs: 100, CRTs: 101, RegionID: 0, KeySpanID: 0, Sequence: 0", raw.String()) require.Equal(t, int64(6), raw.ApproximateDataSize()) } + +func TestRawKVEntryCodec(t *testing.T) { + v := RawKVEntry{ + OpType: OpTypePut, + Key: []byte("key"), + Value: []byte("value"), + OldValue: []byte("old_value"), + StartTs: 0, + CRTs: 1, + ExpiredTs: 2, + RegionID: 3, + KeySpanID: 4, + Sequence: 5, + } + + { + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + vn := RawKVEntry{} + left, err := vn.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + require.Equal(t, v, vn) + } + + { + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRawKVEntry Msgsize() is inaccurate") + } + + vn := RawKVEntry{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + require.Equal(t, v, vn) + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } + } +} diff --git a/cdc/go.mod b/cdc/go.mod index f58745fc..a2889cdf 100644 --- a/cdc/go.mod +++ b/cdc/go.mod @@ -216,4 +216,5 @@ replace ( github.com/go-ldap/ldap/v3 => github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 github.com/uber-go/atomic => go.uber.org/atomic v1.4.0 + sourcegraph.com/sourcegraph/appdash-data => github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) diff --git a/cdc/go.sum b/cdc/go.sum index 9e045cc6..326037f4 100644 --- a/cdc/go.sum +++ b/cdc/go.sum @@ -967,6 +967,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:8ZnTA26bBOoPkAbbitKPgNlpw0Bwt7ZlpYgZWHWJR/w= +github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:tNZjgbYncKL5HxvDULAr/mWDmFz4B7H8yrXEDlnoIiw= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -1731,5 +1733,3 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= -sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/cdc/tools/Makefile b/cdc/tools/Makefile new file mode 100644 index 00000000..39364d64 --- /dev/null +++ b/cdc/tools/Makefile @@ -0,0 +1,61 @@ +### Makefile for TiKV-CDC tools + +tools/bin/failpoint-ctl: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl + +tools/bin/gocovmerge: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/gocovmerge github.com/zhouqiang-cl/gocovmerge + +tools/bin/goveralls: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/goveralls github.com/mattn/goveralls + +tools/bin/golangci-lint: tools/check/go.mod + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b tools/bin v1.55.2 + +tools/bin/mockgen: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/mockgen github.com/golang/mock/mockgen + +tools/bin/protoc-gen-gogofaster: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/protoc-gen-gogofaster github.com/gogo/protobuf/protoc-gen-gogofaster + +tools/bin/protoc-gen-grpc-gateway: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway + +tools/bin/statik: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/statik github.com/rakyll/statik + +gofumpt: + $(GO) install mvdan.cc/gofumpt@latest + +shfmt: + $(GO) install mvdan.cc/sh/v3/cmd/shfmt@latest + +tools/bin/oapi-codegen: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/oapi-codegen github.com/deepmap/oapi-codegen/cmd/oapi-codegen + +gocov: + $(GO) install github.com/axw/gocov/gocov@latest + +gocov-xml: + $(GO) install github.com/AlekSi/gocov-xml@latest + +gotestsum: + $(GO) install gotest.tools/gotestsum@latest + +tools/bin/errdoc-gen: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/errdoc-gen github.com/pingcap/errors/errdoc-gen + +tools/bin/gosec: + curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b tools/bin v2.18.2 + +shellcheck: tools/bin/shellcheck + tools/bin/shellcheck ./**/*.sh + +tools/bin/shellcheck: + wget -qO- "https://github.com/koalaman/shellcheck/releases/download/$(SCVERSION)/shellcheck-$(SCVERSION).$(ARCH).x86_64.tar.xz" | tar -xJv + mv "shellcheck-$(SCVERSION)/shellcheck" tools/bin/ + chmod +x tools/bin/shellcheck + rm -rf "shellcheck-$(SCVERSION)" + +tools/bin/msgp: tools/check/go.mod + cd tools/check && $(GO) build -mod=mod -o ../bin/msgp github.com/tinylib/msgp diff --git a/cdc/tools/check/go.mod b/cdc/tools/check/go.mod index 12e99b09..c9574068 100644 --- a/cdc/tools/check/go.mod +++ b/cdc/tools/check/go.mod @@ -15,6 +15,7 @@ require ( ) require ( + github.com/tinylib/msgp v1.1.9 // indirect gotest.tools/gotestsum v1.8.1 // indirect mvdan.cc/sh/v3 v3.5.1 // indirect ) diff --git a/cdc/tools/check/go.sum b/cdc/tools/check/go.sum index 57119cfe..aaa7261e 100644 --- a/cdc/tools/check/go.sum +++ b/cdc/tools/check/go.sum @@ -505,6 +505,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 h1:8l9lu9RjWkI/VeqrP+Fn3tvZNPu5GYP0rYLLN5Q46go= @@ -615,6 +617,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/tetafro/godot v1.4.9/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8= github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= +github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -735,6 +739,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -879,6 +885,8 @@ golang.org/x/sys v0.0.0-20211031064116-611d5d643895 h1:iaNpwpnrgL5jzWS0vCNnfa8Hq golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= @@ -1004,6 +1012,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=