Skip to content

Commit

Permalink
*: speed up the operation of "admin check table" (#8572) (#11547)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zimulala @sre-bot
zimulala authored and sre-bot committed Aug 5, 2019
1 parent 9cb903e commit 785cb24
Show file tree
Hide file tree
Showing 10 changed files with 662 additions and 194 deletions.
156 changes: 131 additions & 25 deletions executor/admin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)

r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
Expand All @@ -115,7 +115,7 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)
r = tk.MustQuery("admin recover index admin_test c2")
r.Check(testkit.Rows("1 5"))
Expand All @@ -137,9 +137,9 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)

r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
Expand Down Expand Up @@ -261,9 +261,9 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)
r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
r.Check(testkit.Rows("11"))
Expand All @@ -273,9 +273,9 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) {
r.Check(testkit.Rows("6"))
tk.MustExec("admin check index admin_test c2")

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c3")
err = tk.ExecToErr("admin check index admin_test c3")
c.Assert(err, NotNil)
r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c3)")
r.Check(testkit.Rows("9"))
Expand Down Expand Up @@ -322,9 +322,9 @@ func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test `primary`")
err = tk.ExecToErr("admin check index admin_test `primary`")
c.Assert(err, NotNil)
r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(`primary`)")
r.Check(testkit.Rows("6"))
Expand Down Expand Up @@ -374,11 +374,11 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
err = txn.Commit(context.Background())
c.Assert(err, IsNil)

_, err = tk.Exec("admin check table admin_test")
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c1")
err = tk.ExecToErr("admin check index admin_test c1")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_test c2")
err = tk.ExecToErr("admin check index admin_test c2")
c.Assert(err, NotNil)
r := tk.MustQuery("SELECT COUNT(*) FROM admin_test")
r.Check(testkit.Rows("3"))
Expand All @@ -399,6 +399,112 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
tk.MustExec("admin check table admin_test")
}

func (s *testSuite2) TestAdminCheckTableFailed(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists admin_test")
tk.MustExec("create table admin_test (c1 int, c2 int, c3 varchar(255) default '1', primary key(c1), key(c3), unique key(c2), key(c2, c3))")
tk.MustExec("insert admin_test (c1, c2, c3) values (-10, -20, 'y'), (-1, -10, 'z'), (1, 11, 'a'), (2, 12, 'b'), (5, 15, 'c'), (10, 20, 'd'), (20, 30, 'e')")

// Make some corrupted index. Build the index information.
s.ctx = mock.NewContext()
s.ctx.Store = s.store
is := s.domain.InfoSchema()
dbName := model.NewCIStr("test")
tblName := model.NewCIStr("admin_test")
tbl, err := is.TableByName(dbName, tblName)
c.Assert(err, IsNil)
tblInfo := tbl.Meta()
idxInfo := tblInfo.Indices[1]
indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, idxInfo)
sc := s.ctx.GetSessionVars().StmtCtx
tk.Se.GetSessionVars().IndexLookupSize = 3
tk.Se.GetSessionVars().MaxChunkSize = 3

// Reduce one row of index.
// Table count > index count.
// Index c2 is missing 11.
txn, err := s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(-10), -1, nil)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals,
"[executor:8003]admin_test err:[admin:1]index:<nil> != record:&admin.RecordData{Handle:-1, Values:[]types.Datum{types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:-10, b:[]uint8(nil), x:interface {}(nil)}}}")
c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue)
r := tk.MustQuery("admin recover index admin_test c2")
r.Check(testkit.Rows("1 7"))
tk.MustExec("admin check table admin_test")

// Add one row of index.
// Table count < index count.
// Index c2 has one more values ​​than table data: 0, and the handle 0 hasn't correlative record.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(0), 0)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "handle 0, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:0, b:[]uint8(nil), x:interface {}(nil)} != record:<nil>")

// Add one row of index.
// Table count < index count.
// Index c2 has two more values ​​than table data: 10, 13, and these handles have correlative record.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(0), 0, nil)
c.Assert(err, IsNil)
// Make sure the index value "19" is smaller "21". Then we scan to "19" before "21".
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(19), 10)
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(13), 2)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "col c2, handle 2, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:13, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:12, b:[]uint8(nil), x:interface {}(nil)}")

// Table count = index count.
// Two indices have the same handle.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(13), 2, nil)
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(12), 2, nil)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}")

// Table count = index count.
// Index c2 has one line of data is 19, the corresponding table data is 20.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(12), 2)
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(20), 10, nil)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
err = tk.ExecToErr("admin check table admin_test")
c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}")

// Recover records.
txn, err = s.store.Begin()
c.Assert(err, IsNil)
err = indexOpr.Delete(sc, txn, types.MakeDatums(19), 10, nil)
c.Assert(err, IsNil)
_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(20), 10)
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
tk.MustExec("admin check table admin_test")
}

func (s *testSuite1) TestAdminCheckTable(c *C) {
// test NULL value.
tk := testkit.NewTestKit(c, s.store)
Expand Down Expand Up @@ -455,22 +561,22 @@ func (s *testSuite1) TestAdminCheckTable(c *C) {

// Test index in virtual generated column.
tk.MustExec(`drop table if exists test`)
tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')) , index idxc(c));`)
tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')), index idxc(c));`)
tk.MustExec(`INSERT INTO test set b='{"d": 100}';`)
tk.MustExec(`admin check table test;`)
// Test prefix index.
tk.MustExec(`drop table if exists t`)
tk.MustExec(`CREATE TABLE t (
ID CHAR(32) NOT NULL,
name CHAR(32) NOT NULL,
value CHAR(255),
INDEX indexIDname (ID(8),name(8)));`)
ID CHAR(32) NOT NULL,
name CHAR(32) NOT NULL,
value CHAR(255),
INDEX indexIDname (ID(8),name(8)));`)
tk.MustExec(`INSERT INTO t VALUES ('keyword','urlprefix','text/ /text');`)
tk.MustExec(`admin check table t;`)

tk.MustExec("use mysql")
tk.MustExec(`admin check table test.t;`)
_, err := tk.Exec("admin check table t")
err := tk.ExecToErr("admin check table t")
c.Assert(err, NotNil)

// test add index on time type column which have default value
Expand Down Expand Up @@ -510,7 +616,7 @@ func (s *testSuite1) TestAdminCheckTable(c *C) {
tk.MustExec(`drop table if exists t1`)
tk.MustExec(`create table t1 (a decimal(2,1), index(a))`)
tk.MustExec(`insert into t1 set a='1.9'`)
_, err = tk.Exec(`alter table t1 modify column a decimal(3,2);`)
err = tk.ExecToErr(`alter table t1 modify column a decimal(3,2);`)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify decimal column precision")
tk.MustExec(`delete from t1;`)
Expand Down Expand Up @@ -553,9 +659,9 @@ func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) {
c.Assert(err, IsNil)
err = txn.Commit(context.Background())
c.Assert(err, IsNil)
_, err = tk.Exec("admin check table admin_t_s")
err = tk.ExecToErr("admin check table admin_t_s")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_t_s a")
err = tk.ExecToErr("admin check index admin_t_s a")
c.Assert(err, NotNil)

// For mocktikv, safe point is not initialized, we manually insert it for snapshot to use.
Expand All @@ -572,9 +678,9 @@ func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) {
tk.MustExec("admin check index admin_t_s a;")

tk.MustExec("set @@tidb_snapshot = ''")
_, err = tk.Exec("admin check table admin_t_s")
err = tk.ExecToErr("admin check table admin_t_s")
c.Assert(err, NotNil)
_, err = tk.Exec("admin check index admin_t_s a")
err = tk.ExecToErr("admin check index admin_t_s a")
c.Assert(err, NotNil)

r := tk.MustQuery("admin cleanup index admin_t_s a")
Expand Down
55 changes: 51 additions & 4 deletions executor/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,8 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor {
b.err = err
return nil
}
readerExec.ranges = ranger.FullRange()
readerExec.isCheckOp = true

buildIndexLookUpChecker(b, v.IndexLookUpReader, readerExec)

e := &CheckIndexExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()),
Expand All @@ -320,12 +320,59 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor {
return e
}

// buildIndexLookUpChecker builds check information to IndexLookUpReader.
func buildIndexLookUpChecker(b *executorBuilder, readerPlan *plannercore.PhysicalIndexLookUpReader,
readerExec *IndexLookUpExecutor) {
is := readerPlan.IndexPlans[0].(*plannercore.PhysicalIndexScan)
readerExec.dagPB.OutputOffsets = make([]uint32, 0, len(is.Index.Columns))
for i := 0; i <= len(is.Index.Columns); i++ {
readerExec.dagPB.OutputOffsets = append(readerExec.dagPB.OutputOffsets, uint32(i))
}
readerExec.ranges = ranger.FullRange()
ts := readerPlan.TablePlans[0].(*plannercore.PhysicalTableScan)
readerExec.handleIdx = ts.HandleIdx

tps := make([]*types.FieldType, 0, len(is.Columns)+1)
for _, col := range is.Columns {
tps = append(tps, &col.FieldType)
}
tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
readerExec.checkIndexValue = &checkIndexValue{genExprs: is.GenExprs, idxColTps: tps}

colNames := make([]string, 0, len(is.Columns))
for _, col := range is.Columns {
colNames = append(colNames, col.Name.O)
}
var err error
readerExec.idxTblCols, err = table.FindCols(readerExec.table.Cols(), colNames, true)
if err != nil {
b.err = errors.Trace(err)
return
}
}

func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) Executor {
readerExecs := make([]*IndexLookUpExecutor, 0, len(v.IndexLookUpReaders))
for _, readerPlan := range v.IndexLookUpReaders {
readerExec, err := buildNoRangeIndexLookUpReader(b, readerPlan)
if err != nil {
b.err = errors.Trace(err)
return nil
}
buildIndexLookUpChecker(b, readerPlan, readerExec)

readerExecs = append(readerExecs, readerExec)
}

e := &CheckTableExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()),
tables: v.Tables,
dbName: v.DBName,
tblInfo: v.TblInfo,
indices: v.Indices,
is: b.is,
genExprs: v.GenExprs,
srcs: readerExecs,
exitCh: make(chan struct{}),
retCh: make(chan error, len(v.Indices)),
}
return e
}
Expand Down
Loading

0 comments on commit 785cb24

Please sign in to comment.