Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ddl: add table option pre_split_regions for pre-split region when create table with shard_row_id_bits. #10138

Merged
merged 22 commits into from
Apr 24, 2019
Merged

ddl: add table option pre_split_regions for pre-split region when create table with shard_row_id_bits. #10138

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
5c613ad
init
crazycs520 Apr 10, 2019
2bc50c8
add scatter
crazycs520 Apr 12, 2019
a88ea9b
fix test
crazycs520 Apr 12, 2019
3145795
only split region for positive num
crazycs520 Apr 12, 2019
2fccfab
add PreSplitRegions option
crazycs520 Apr 12, 2019
77f0a96
add retry for scatter region
crazycs520 Apr 12, 2019
189335d
refine comment
crazycs520 Apr 12, 2019
03783c0
Merge branch 'master' into splite-region
crazycs520 Apr 12, 2019
b517f03
remove blank line
crazycs520 Apr 15, 2019
f96d8d3
Merge branch 'master' into splite-region
crazycs520 Apr 16, 2019
55cd7be
update go.mod for pd
crazycs520 Apr 16, 2019
eb27dfb
add wait scatter finish
crazycs520 Apr 17, 2019
a70696f
add table option: wait_split_finish
crazycs520 Apr 17, 2019
df45ac4
Merge branch 'master' of https://github.com/pingcap/tidb into splite-…
crazycs520 Apr 18, 2019
e362d68
split and scatter region first, wait scatter finish together
crazycs520 Apr 18, 2019
b909b0a
enlarge scatter back off time and add comment
crazycs520 Apr 22, 2019
4cc122b
show pre_split_regions when show create table
crazycs520 Apr 23, 2019
0cb2a3a
add tidb_wait_table_split_finish variable
crazycs520 Apr 24, 2019
1bd3163
Merge branch 'master' of https://github.com/pingcap/tidb into splite-…
crazycs520 Apr 24, 2019
a553dc2
update go.mod
crazycs520 Apr 24, 2019
825729c
make tidy
crazycs520 Apr 24, 2019
2fce640
remove and add bland line
crazycs520 Apr 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ddl/ddl_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1569,8 +1569,13 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err
tbInfo.ShardRowIDBits = shardRowIDBitsMax
}
tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits
case ast.TableOptionPreSplitRegion:
tbInfo.PreSplitRegions = op.UintValue
}
}
if tbInfo.PreSplitRegions > tbInfo.ShardRowIDBits {
tbInfo.PreSplitRegions = tbInfo.ShardRowIDBits
}

return nil
}
Expand Down
34 changes: 34 additions & 0 deletions ddl/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,11 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
// TODO: Add restrictions to this operation.
go splitTableRegion(d.store, tbInfo.ID)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we modify splitTableRegion strategy here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR is not to do this.

}

if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 {
go preSplitTableRegion(d.store, tbInfo)
}

// Finish this job.
job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
asyncNotifyEvent(d, &util.Event{Tp: model.ActionCreateTable, TableInfo: tbInfo})
Expand Down Expand Up @@ -321,6 +326,7 @@ func checkSafePoint(w *worker, snapshotTS uint64) error {

type splitableStore interface {
SplitRegion(splitKey kv.Key) error
SplitRegionAndScatter(splitKey kv.Key) error
}

func splitTableRegion(store kv.Storage, tableID int64) {
Expand All @@ -335,6 +341,34 @@ func splitTableRegion(store kv.Storage, tableID int64) {
}
}

func preSplitTableRegion(store kv.Storage, tblInfo *model.TableInfo) {
s, ok := store.(splitableStore)
if !ok {
return
}
// Split table region.
step := int64(1 << (tblInfo.ShardRowIDBits - tblInfo.PreSplitRegions))
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
// The highest bit is the symbol bit， and alloc _tidb_rowid will always be positive number.
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
// So we only need to split the region for the positive number.
max := int64(1 << (tblInfo.ShardRowIDBits - 1))
for p := int64(step); p < max; p += step {
recordID := p << (64 - tblInfo.ShardRowIDBits)
nolouch marked this conversation as resolved.
Show resolved Hide resolved
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
recordPrefix := tablecodec.GenTableRecordPrefix(tblInfo.ID)
key := tablecodec.EncodeRecordKey(recordPrefix, recordID)
if err := s.SplitRegionAndScatter(key); err != nil {
logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err))
}
}

// Split index region.
for _, idx := range tblInfo.Indices {
indexPrefix := tablecodec.EncodeTableIndexPrefix(tblInfo.ID, idx.ID)
if err := s.SplitRegionAndScatter(indexPrefix); err != nil {
logutil.Logger(ddlLogCtx).Warn("[ddl] pre split table index region failed", zap.String("index", idx.Name.L), zap.Error(err))
}
}
}

func getTable(store kv.Storage, schemaID int64, tblInfo *model.TableInfo) (table.Table, error) {
alloc := autoid.NewAllocator(store, tblInfo.GetDBID(schemaID), tblInfo.IsAutoIncColUnsigned())
tbl, err := table.TableFromMeta(alloc, tblInfo)
Expand Down
25 changes: 6 additions & 19 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,28 @@ module github.com/pingcap/tidb
require (
github.com/BurntSushi/toml v0.3.1
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d
github.com/boltdb/bolt v1.3.1 // indirect
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/coreos/bbolt v1.3.0 // indirect
github.com/coreos/etcd v3.3.10+incompatible
github.com/coreos/go-semver v0.2.0 // indirect
github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect
github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4
github.com/gogo/protobuf v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
github.com/golang/protobuf v1.2.0
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/mux v1.6.2
github.com/gorilla/websocket v1.4.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect
github.com/jonboulle/clockwork v0.1.0 // indirect
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect
github.com/myesui/uuid v1.0.0 // indirect
Expand All @@ -44,18 +34,16 @@ require (
github.com/onsi/gomega v1.4.3 // indirect
github.com/opentracing/basictracer-go v1.0.0
github.com/opentracing/opentracing-go v1.0.2
github.com/philhofer/fwd v1.0.0 // indirect
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
github.com/pingcap/errors v0.11.1
github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e
github.com/pingcap/kvproto v0.0.0-20190215154024-7f2fc73ef562
github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d
github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596
github.com/pingcap/parser v0.0.0-20190409044748-a0b301443a30
github.com/pingcap/pd v2.1.0-rc.4+incompatible
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible
github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7
github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 // indirect
github.com/prometheus/client_golang v0.9.0
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect
Expand All @@ -65,19 +53,14 @@ require (
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 // indirect
github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca // indirect
github.com/sirupsen/logrus v1.2.0
github.com/soheilhy/cmux v0.1.4 // indirect
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72
github.com/struCoder/pidusage v0.1.2
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tinylib/msgp v1.1.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
github.com/twinj/uuid v1.0.0
github.com/uber-go/atomic v1.3.2 // indirect
github.com/uber/jaeger-client-go v2.15.0+incompatible
github.com/uber/jaeger-lib v1.5.0 // indirect
github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 // indirect
github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d // indirect
github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect
go.uber.org/zap v1.9.1
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb // indirect
Expand All @@ -90,3 +73,7 @@ require (
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)

replace github.com/pingcap/pd => github.com/nolouch/pd v0.0.0-20190411093849-8f8de63fdb9f

replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190412094707-fc884a4502bc
Loading