Skip to content

Commit

Permalink
Merge branch 'master' into fix-issue-28804
Browse files Browse the repository at this point in the history
  • Loading branch information
tiancaiamao committed Oct 26, 2021
2 parents cf09f32 + 88f5711 commit 85de026
Show file tree
Hide file tree
Showing 445 changed files with 35,412 additions and 13,801 deletions.
1 change: 1 addition & 0 deletions .github/licenserc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ header:
- 'LICENSE'
- '.github/'
- 'parser/'
- 'dumpling/'
comment: on-failure
20 changes: 9 additions & 11 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
<!-- Thank you for contributing to TiDB!
<!--
Thank you for contributing to TiDB!
PR Title Format:
1. pkg [, pkg2, pkg3]: what's changed
Expand All @@ -8,19 +10,13 @@ PR Title Format:

### What problem does this PR solve?

Issue Number: close #xxx <!-- REMOVE this line if no issue to close -->
Issue Number: close #xxx

Problem Summary:

### What is changed and how it works?

Proposal: [xxx](url) <!-- REMOVE this line if not applicable -->

What's Changed:

How it Works:

### Check List <!--REMOVE the items that are not applicable-->
### Check List

Tests <!-- At least one of them must be included. -->

Expand All @@ -43,8 +39,10 @@ Documentation
- [ ] Contains experimental features
- [ ] Changes MySQL compatibility

### Release note <!-- bugfixes or new feature need a release note -->
### Release note

<!-- bugfix or new feature needs a release note -->

```release-note
Please add a release note, or a 'None' if it is not needed.
None
```
157 changes: 157 additions & 0 deletions .github/workflows/dumpling_integration_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
name: Dumpling
on:
push:
branches:
- master
- release-*
paths:
- 'dumpling/**'
- 'br/pkg/storage/**'
- 'br/pkg/utils/**'
- 'br/pkg/summary/**'
- 'store/helper/**'
- 'tablecodec/**'
- 'util/codec/**'
- 'parser/model/**'
pull_request:
branches:
- master
- release-*
paths:
- 'dumpling/**'
- 'br/pkg/storage/**'
- 'br/pkg/utils/**'
- 'br/pkg/summary/**'
- 'store/helper/**'
- 'tablecodec/**'
- 'util/codec/**'
- 'parser/model/**'

jobs:
unit-test:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: true
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: 1.16
- name: Unit test
run: make dumpling_unit_test WITH_RACE=1
- uses: codecov/codecov-action@v1

integration-test-mysql-5735:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: true
services:
mysql:
image: mysql:5.7.35
env:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- uses: actions/checkout@v2
- name: Shutdown Ubuntu MySQL (SUDO)
run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: 1.16
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Get dependencies
run: go mod download
- name: Download dependencies
run: sh dumpling/install.sh
- name: Integration test
run: make dumpling_integration_test
- name: Set up tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3

integration-test-mysql-8026:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: true
services:
mysql:
image: mysql:8.0.26
env:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- uses: actions/checkout@v2
- name: Shutdown Ubuntu MySQL (SUDO)
run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: 1.16
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Get dependencies
run: go mod download
- name: Download dependencies
run: sh dumpling/install.sh
- name: Integration test
run: make dumpling_integration_test
- name: Set up tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3

integration-test-mysql-8022:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: true
services:
mysql:
image: mysql:8.0.22
env:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- uses: actions/checkout@v2
- name: Shutdown Ubuntu MySQL (SUDO)
run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: 1.16
- uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Get dependencies
run: go mod download
- name: Download dependencies
run: sh dumpling/install.sh
- name: Integration test
run: make dumpling_integration_test
- name: Set up tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,7 @@ vendor
.DS_Store
.vscode
bench_daily.json
coverage.txt
var
fix.sql
export-20*/
6 changes: 6 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ linters:
- staticcheck
- stylecheck
- gosec
- asciicheck
- bodyclose
- rowserrcheck
- unconvert

linters-settings:
staticcheck:
checks: ["S1002","S1004","S1007","S1009","S1010","S1012","S1019","S1020","S1021","S1024","S1030","SA2*","SA3*","SA4009","SA5*","SA6000","SA6001","SA6005", "-SA2002"]
Expand All @@ -30,3 +35,4 @@ issues:
linters:
- errcheck
- gosec
- rowserrcheck
83 changes: 44 additions & 39 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

include Makefile.common

.PHONY: all clean test gotest server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl
.PHONY: all clean test gotest server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling

default: server buildsucc

Expand Down Expand Up @@ -44,7 +44,7 @@ goword:tools/bin/goword
tools/bin/goword $(FILES) 2>&1 | $(FAIL_ON_STDOUT)

check-static: tools/bin/golangci-lint
tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES_WITHOUT_BR))
tools/bin/golangci-lint run -v $$($(PACKAGE_DIRECTORIES_TIDB_TESTS))

unconvert:tools/bin/unconvert
@echo "unconvert check(skip check the genenrated or copied code in lightning)"
Expand All @@ -60,11 +60,11 @@ errdoc:tools/bin/errdoc-gen

lint:tools/bin/revive
@echo "linting"
@tools/bin/revive -formatter friendly -config tools/check/revive.toml $(FILES_WITHOUT_BR)
@tools/bin/revive -formatter friendly -config tools/check/revive.toml $(FILES_TIDB_TESTS)

vet:
@echo "vet"
$(GO) vet -all $(PACKAGES_WITHOUT_BR) 2>&1 | $(FAIL_ON_STDOUT)
$(GO) vet -all $(PACKAGES_TIDB_TESTS) 2>&1 | $(FAIL_ON_STDOUT)

tidy:
@echo "go mod tidy"
Expand All @@ -83,7 +83,7 @@ test: test_part_1 test_part_2

test_part_1: checklist explaintest

test_part_2: test_part_parser gotest gogenerate br_unit_test
test_part_2: test_part_parser gotest gogenerate br_unit_test dumpling_unit_test

test_part_parser: parser_yacc test_part_parser_dev

Expand All @@ -100,6 +100,8 @@ parser_unit_test:

test_part_br: br_unit_test br_integration_test

test_part_dumpling: dumpling_unit_test dumpling_integration_test

explaintest: server_check
@cd cmd/explaintest && ./run-tests.sh -s ../../bin/tidb-server

Expand All @@ -114,44 +116,19 @@ ifeq ("$(TRAVIS_COVERAGE)", "1")
endif

devgotest: failpoint-enable
ifeq ("$(TRAVIS_COVERAGE)", "1")
@echo "Running in TRAVIS_COVERAGE mode."
$(GO) get github.com/go-playground/overalls
@export log_level=info; \
$(OVERALLS) -project=github.com/pingcap/tidb \
-covermode=count \
-ignore='.git,br,vendor,cmd,docs,tests,LICENSES' \
-concurrency=4 \
-- -coverpkg=./... \
|| { $(FAILPOINT_DISABLE); exit 1; }
else
# grep regex: Filter out all tidb logs starting with:
# - '[20' (like [2021/09/15 ...] [INFO]..)
# - 'PASS:' to ignore passed tests
# - 'ok ' to ignore passed directories
@echo "Running in native mode."
@export log_level=info; export TZ='Asia/Shanghai'; \
$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_WITHOUT_BR) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
endif
$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
@$(FAILPOINT_DISABLE)


gotest: failpoint-enable
ifeq ("$(TRAVIS_COVERAGE)", "1")
@echo "Running in TRAVIS_COVERAGE mode."
$(GO) get github.com/go-playground/overalls
@export log_level=info; \
$(OVERALLS) -project=github.com/pingcap/tidb \
-covermode=count \
-ignore='.git,br,vendor,cmd,docs,tests,LICENSES' \
-concurrency=4 \
-- -coverpkg=./... \
|| { $(FAILPOINT_DISABLE); exit 1; }
else
@echo "Running in native mode."
@export log_level=info; export TZ='Asia/Shanghai'; \
$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_WITHOUT_BR) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
endif
$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
@$(FAILPOINT_DISABLE)

race: failpoint-enable
Expand Down Expand Up @@ -277,13 +254,14 @@ endif
# Usage:
# make bench-daily TO=/path/to/file.json
bench-daily:
go test github.com/pingcap/tidb/session -run TestBenchDaily --outfile bench_daily.json
go test github.com/pingcap/tidb/executor -run TestBenchDaily --outfile bench_daily.json
go test github.com/pingcap/tidb/tablecodec -run TestBenchDaily --outfile bench_daily.json
go test github.com/pingcap/tidb/expression -run TestBenchDaily --outfile bench_daily.json
go test github.com/pingcap/tidb/util/rowcodec -run TestBenchDaily --outfile bench_daily.json
go test github.com/pingcap/tidb/util/codec -run TestBenchDaily --outfile bench_daily.json
go test github.com/pingcap/tidb/util/benchdaily -run TestBenchDaily \
go test github.com/pingcap/tidb/session -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/executor -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/tablecodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/expression -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/util/rowcodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/util/codec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/distsql -run TestBenchDaily -bench Ignore --outfile bench_daily.json
go test github.com/pingcap/tidb/util/benchdaily -run TestBenchDaily -bench Ignore \
-date `git log -n1 --date=unix --pretty=format:%cd` \
-commit `git log -n1 --pretty=format:%h` \
-outfile $(TO)
Expand Down Expand Up @@ -382,3 +360,30 @@ br_bins:
data_parsers: tools/bin/vfsgendev br/pkg/lightning/mydump/parser_generated.go br_web
PATH="$(GOPATH)/bin":"$(PATH)":"$(TOOLS)" protoc -I. -I"$(GOPATH)/src" br/pkg/lightning/checkpoints/checkpointspb/file_checkpoints.proto --gogofaster_out=.
tools/bin/vfsgendev -source='"github.com/pingcap/tidb/br/pkg/lightning/web".Res' && mv res_vfsdata.go br/pkg/lightning/web/

build_dumpling:
$(DUMPLING_GOBUILD) $(RACE_FLAG) -tags codes -o $(DUMPLING_BIN) dumpling/cmd/dumpling/main.go

dumpling_unit_test: export DUMPLING_ARGS=$$($(DUMPLING_PACKAGES))
dumpling_unit_test: failpoint-enable
$(DUMPLING_GOTEST) $(RACE_FLAG) -coverprofile=coverage.txt -covermode=atomic -tags leak $(DUMPLING_ARGS) || ( make failpoint-disable && exit 1 )
@make failpoint-disable

dumpling_integration_test: dumpling_bins failpoint-enable build_dumpling
@make failpoint-disable
./dumpling/tests/run.sh $(CASE)

dumpling_tools:
@echo "install dumpling tools..."
@cd dumpling/tools && make

dumpling_tidy:
@echo "go mod tidy"
GO111MODULE=on go mod tidy
git diff --exit-code go.mod go.sum dumpling/tools/go.mod dumpling/tools/go.sum

dumpling_bins:
@which bin/tidb-server
@which bin/minio
@which bin/tidb-lightning
@which bin/sync_diff_inspector
Loading

0 comments on commit 85de026

Please sign in to comment.