Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: speed up the operation of "admin check table" (#11568, #8572) #11547

Merged
merged 5 commits into from
Aug 5, 2019
Merged

*: speed up the operation of "admin check table" (#11568, #8572) #11547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ea09111
*: speed up the operation of "admin check table" (#8572)
zimulala Jul 31, 2019
7b52335
executor: fix data race of "admin check table" (#11568)
zimulala Aug 1, 2019
8fb0799
Merge branch 'release-3.0' into zimuxia/check-table-3
zimulala Aug 2, 2019
48b7fa4
Merge branch 'release-3.0' into check-table-3
sre-bot Aug 4, 2019
8c1e8c7
Merge branch 'release-3.0' into check-table-3
zz-jason Aug 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 131 additions & 25 deletions executor/admin_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)

r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
Expand All @@ -115,7 +115,7 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)
r = tk.MustQuery("admin recover index admin_test c2")
r.Check(testkit.Rows("1 5"))
Expand All @@ -137,9 +137,9 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)

r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
Expand Down Expand Up @@ -261,9 +261,9 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)
r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
r.Check(testkit.Rows("11"))
Expand All @@ -273,9 +273,9 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) {
r.Check(testkit.Rows("6"))
tk.MustExec("admin check index admin_test c2")

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c3")
err = tk.ExecToErr("admin check index admin_test c3")
c.Assert(err, NotNil)
r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c3)")
r.Check(testkit.Rows("9"))
Expand Down Expand Up @@ -322,9 +322,9 @@ func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test `primary`")
err = tk.ExecToErr("admin check index admin_test `primary`")
c.Assert(err, NotNil)
r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(`primary`)")
r.Check(testkit.Rows("6"))
Expand Down Expand Up @@ -374,11 +374,11 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c1")
err = tk.ExecToErr("admin check index admin_test c1")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)
r := tk.MustQuery("SELECT COUNT(*) FROM admin_test")
r.Check(testkit.Rows("3"))
Expand All @@ -399,6 +399,112 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
tk.MustExec("admin check table admin_test")
}

func (s *testSuite2) TestAdminCheckTableFailed(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists admin_test")
tk.MustExec("create table admin_test (c1 int, c2 int, c3 varchar(255) default '1', primary key(c1), key(c3), unique key(c2), key(c2, c3))")
tk.MustExec("insert admin_test (c1, c2, c3) values (-10, -20, 'y'), (-1, -10, 'z'), (1, 11, 'a'), (2, 12, 'b'), (5, 15, 'c'), (10, 20, 'd'), (20, 30, 'e')")

// Make some corrupted index. Build the index information.
s.ctx = mock.NewContext()
s.ctx.Store = s.store
is := s.domain.InfoSchema()
dbName := model.NewCIStr("test")
tblName := model.NewCIStr("admin_test")
tbl, err := is.TableByName(dbName, tblName)
c.Assert(err, IsNil)
tblInfo := tbl.Meta()
idxInfo := tblInfo.Indices[1]
indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, idxInfo)
sc := s.ctx.GetSessionVars().StmtCtx
tk.Se.GetSessionVars().IndexLookupSize = 3
tk.Se.GetSessionVars().MaxChunkSize = 3

// Reduce one row of index.
// Table count > index count.
// Index c2 is missing 11.
txn, err := s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(-10), -1, nil)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals,
"[executor:8003]admin_test err:[admin:1]index:<nil> != record:&admin.RecordData{Handle:-1, Values:[]types.Datum{types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:-10, b:[]uint8(nil), x:interface {}(nil)}}}")
c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue)
r := tk.MustQuery("admin recover index admin_test c2")
r.Check(testkit.Rows("1 7"))
tk.MustExec("admin check table admin_test")

// Add one row of index.
// Table count < index count.
// Index c2 has one more values ​​than table data: 0, and the handle 0 hasn't correlative record.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(0), 0)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "handle 0, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:0, b:[]uint8(nil), x:interface {}(nil)} != record:<nil>")

// Add one row of index.
// Table count < index count.
// Index c2 has two more values ​​than table data: 10, 13, and these handles have correlative record.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(0), 0, nil)
c.Assert(err, IsNil)
// Make sure the index value "19" is smaller "21". Then we scan to "19" before "21".
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(19), 10)
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(13), 2)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "col c2, handle 2, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:13, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:12, b:[]uint8(nil), x:interface {}(nil)}")

// Table count = index count.
// Two indices have the same handle.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(13), 2, nil)
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(12), 2, nil)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}")

// Table count = index count.
// Index c2 has one line of data is 19, the corresponding table data is 20.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(12), 2)
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(20), 10, nil)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}")

// Recover records.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(19), 10, nil)
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(20), 10)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
tk.MustExec("admin check table admin_test")
}

func (s *testSuite1) TestAdminCheckTable(c *C) {
// test NULL value.
tk := testkit.NewTestKit(c, s.store)
Expand Down Expand Up @@ -455,22 +561,22 @@ func (s *testSuite1) TestAdminCheckTable(c *C) {

// Test index in virtual generated column.
tk.MustExec(`drop table if exists test`)
tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')) , index idxc(c));`)
tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')), index idxc(c));`)
tk.MustExec(`INSERT INTO test set b='{"d": 100}';`)
tk.MustExec(`admin check table test;`)
// Test prefix index.
tk.MustExec(`drop table if exists t`)
tk.MustExec(`CREATE TABLE t (
ID CHAR(32) NOT NULL,
name CHAR(32) NOT NULL,
value CHAR(255),
INDEX indexIDname (ID(8),name(8)));`)
ID CHAR(32) NOT NULL,
name CHAR(32) NOT NULL,
value CHAR(255),
INDEX indexIDname (ID(8),name(8)));`)
tk.MustExec(`INSERT INTO t VALUES ('keyword','urlprefix','text/ /text');`)
tk.MustExec(`admin check table t;`)

tk.MustExec("use mysql")
tk.MustExec(`admin check table test.t;`)
_, err := tk.Exec("admin check table t")
err := tk.ExecToErr("admin check table t")
c.Assert(err, NotNil)

// test add index on time type column which have default value
Expand Down Expand Up @@ -510,7 +616,7 @@ func (s *testSuite1) TestAdminCheckTable(c *C) {
tk.MustExec(`drop table if exists t1`)
tk.MustExec(`create table t1 (a decimal(2,1), index(a))`)
tk.MustExec(`insert into t1 set a='1.9'`)
_, err = tk.Exec(`alter table t1 modify column a decimal(3,2);`)
err = tk.ExecToErr(`alter table t1 modify column a decimal(3,2);`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify decimal column precision")
tk.MustExec(`delete from t1;`)
Expand Down Expand Up @@ -553,9 +659,9 @@ func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) {
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
_, err = tk.Exec("admin check table admin_t_s")
err = tk.ExecToErr("admin check table admin_t_s")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_t_s a")
err = tk.ExecToErr("admin check index admin_t_s a")
c.Assert(err, NotNil)

// For mocktikv, safe point is not initialized, we manually insert it for snapshot to use.
Expand All @@ -572,9 +678,9 @@ func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) {
tk.MustExec("admin check index admin_t_s a;")

tk.MustExec("set @@tidb_snapshot = ''")
_, err = tk.Exec("admin check table admin_t_s")
err = tk.ExecToErr("admin check table admin_t_s")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_t_s a")
err = tk.ExecToErr("admin check index admin_t_s a")
c.Assert(err, NotNil)

r := tk.MustQuery("admin cleanup index admin_t_s a")
Expand Down
55 changes: 51 additions & 4 deletions executor/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,8 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor {
b.err = err
return nil
}
readerExec.ranges = ranger.FullRange()
readerExec.isCheckOp = true

buildIndexLookUpChecker(b, v.IndexLookUpReader, readerExec)

e := &CheckIndexExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()),
Expand All @@ -320,12 +320,59 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor {
return e
}

// buildIndexLookUpChecker builds check information to IndexLookUpReader.
func buildIndexLookUpChecker(b *executorBuilder, readerPlan *plannercore.PhysicalIndexLookUpReader,
readerExec *IndexLookUpExecutor) {
is := readerPlan.IndexPlans[0].(*plannercore.PhysicalIndexScan)
readerExec.dagPB.OutputOffsets = make([]uint32, 0, len(is.Index.Columns))
for i := 0; i <= len(is.Index.Columns); i++ {
readerExec.dagPB.OutputOffsets = append(readerExec.dagPB.OutputOffsets, uint32(i))
}
readerExec.ranges = ranger.FullRange()
ts := readerPlan.TablePlans[0].(*plannercore.PhysicalTableScan)
readerExec.handleIdx = ts.HandleIdx

tps := make([]*types.FieldType, 0, len(is.Columns)+1)
for _, col := range is.Columns {
tps = append(tps, &col.FieldType)
}
tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
readerExec.checkIndexValue = &checkIndexValue{genExprs: is.GenExprs, idxColTps: tps}

colNames := make([]string, 0, len(is.Columns))
for _, col := range is.Columns {
colNames = append(colNames, col.Name.O)
}
var err error
readerExec.idxTblCols, err = table.FindCols(readerExec.table.Cols(), colNames, true)
if err != nil {
b.err = errors.Trace(err)
return
}
}

func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) Executor {
readerExecs := make([]*IndexLookUpExecutor, 0, len(v.IndexLookUpReaders))
for _, readerPlan := range v.IndexLookUpReaders {
readerExec, err := buildNoRangeIndexLookUpReader(b, readerPlan)
if err != nil {
b.err = errors.Trace(err)
return nil
}
buildIndexLookUpChecker(b, readerPlan, readerExec)

readerExecs = append(readerExecs, readerExec)
}

e := &CheckTableExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()),
tables: v.Tables,
dbName: v.DBName,
tblInfo: v.TblInfo,
indices: v.Indices,
is: b.is,
genExprs: v.GenExprs,
srcs: readerExecs,
exitCh: make(chan struct{}),
retCh: make(chan error, len(v.Indices)),
}
return e
}
Expand Down
Loading