Merge branch 'feature-add-udf-support' of https://github.com/JustProj…

…ect/tidb into feature-add-udf-support

* 'feature-add-udf-support' of https://github.com/JustProject/tidb: (26 commits)
  *: fix bug that the kill command doesn't work when the killed session is waiting for the pessimistic lock (pingcap#12852)
  executor: fix the projection upon the indexLookUp in indexLookUpJoin can't get result. (pingcap#12889)
  planner, executor: support create view on union (pingcap#12595)
  planner/cascades: introduce TransformationID in cascades planner (pingcap#12879)
  executor: fix data race in test (pingcap#12910)
  executor: reuse chunk row for insert on duplicate update (pingcap#12847)
  ddl: speed up tests (pingcap#12888)
  executor: speed up test (pingcap#12896)
  expression: implement vectorized evaluation for `builtinSecondSig` (pingcap#12886)
  expression: implement vectorized evaluation for `builtinJSONObjectSig` (pingcap#12663)
  expression: speed up builtinRepeatSig by using MergeNulls (pingcap#12674)
  expression: speed up unit tests under the expression package (pingcap#12887)
  store,kv: snapshot doesn't cache the non-exists kv entries lead to poor 'insert ignore' performance (pingcap#12872)
  executor: fix data race in `GetDirtyTable()` (pingcap#12767)
  domain: increase TTL to reduce the occurrence of reporting min startTS errors (pingcap#12578)
  executor: split test for speed up (pingcap#12881)
  executor: fix inconsistent of grants privileges with MySQL when executing `grant all on ...` (pingcap#12330)
  expression: implement vectorized evaluation for `builtinJSONUnquoteSig` (pingcap#12841)
  tune grpc connection count between tidb and tikv (pingcap#12884)
  Makefile: change test parallel to 8 (pingcap#12885)
  ...

Makefile

@@ -14,7 +14,7 @@ export PATH := $(path_to_add):$(PATH)
 GO              := GO111MODULE=on go
 GOBUILD         := $(GO) build $(BUILD_FLAG) -tags codes
 GOBUILDCOVERAGE := GOPATH=$(GOPATH) cd tidb-server; $(GO) test -coverpkg="../..." -c .
-GOTEST          := $(GO) test -p 4
+GOTEST          := $(GO) test -p 8
 OVERALLS        := GO111MODULE=on overalls
 
 ARCH      := "`uname -s`"

config/config.go

@@ -404,7 +404,7 @@ var defaultConf = Config{
 		Reporter: OpenTracingReporter{},
 	},
 	TiKVClient: TiKVClient{
-		GrpcConnectionCount:  16,
+		GrpcConnectionCount:  4,
 		GrpcKeepAliveTime:    10,
 		GrpcKeepAliveTimeout: 3,
 		CommitTimeout:        "41s",

config/config.toml.example

@@ -259,7 +259,7 @@ local-agent-host-port = ""
 
 [tikv-client]
 # Max gRPC connections that will be established with each tikv-server.
-grpc-connection-count = 16
+grpc-connection-count = 4
 
 # After a duration of this time in seconds if the client doesn't see any activity it pings
 # the server to see if the transport is still alive.

ddl/db_integration_test.go

@@ -69,6 +69,11 @@ func setupIntegrationSuite(s *testIntegrationSuite, c *C) {
 	s.lease = 50 * time.Millisecond
 	ddl.WaitTimeWhenErrorOccured = 0
 
+	cfg := config.GetGlobalConfig()
+	newCfg := *cfg
+	newCfg.Log.SlowThreshold = 10000
+	config.StoreGlobalConfig(&newCfg)
+
 	s.cluster = mocktikv.NewCluster()
 	mocktikv.BootstrapWithSingleStore(s.cluster)
 	s.mvccStore = mocktikv.MustNewMVCCStore()

ddl/db_partition_test.go

@@ -1248,9 +1248,7 @@ func (s *testIntegrationSuite2) TestPartitionCancelAddIndex(c *C) {
 	base := defaultBatchSize * 2
 	count := base
 	// add some rows
-	for i := 0; i < count; i++ {
-		tk.MustExec("insert into t1 values (?, ?, ?)", i, i, i)
-	}
+	batchInsert(s.tk, "t1", 0, count)
 
 	var checkErr error
 	var c3IdxInfo *model.IndexInfo

ddl/db_test.go

@@ -94,6 +94,7 @@ func setUpSuite(s *testDBSuite, c *C) {
 	cfg := config.GetGlobalConfig()
 	newCfg := *cfg
 	newCfg.EnableTableLock = true
+	newCfg.Log.SlowThreshold = 10000
 	config.StoreGlobalConfig(&newCfg)
 
 	s.cluster = mocktikv.NewCluster()
@@ -225,6 +226,17 @@ func backgroundExec(s kv.Storage, sql string, done chan error) {
 	done <- errors.Trace(err)
 }
 
+func batchInsert(tk *testkit.TestKit, tbl string, start, end int) {
+	dml := fmt.Sprintf("insert into %s values", tbl)
+	for i := start; i < end; i++ {
+		dml += fmt.Sprintf("(%d, %d, %d)", i, i, i)
+		if i != end-1 {
+			dml += ","
+		}
+	}
+	tk.MustExec(dml)
+}
+
 func (s *testDBSuite2) TestAddUniqueIndexRollback(c *C) {
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.mustExec(c, "use test_db")
@@ -234,9 +246,7 @@ func (s *testDBSuite2) TestAddUniqueIndexRollback(c *C) {
 	base := defaultBatchSize * 2
 	count := base
 	// add some rows
-	for i := 0; i < count; i++ {
-		s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
-	}
+	batchInsert(s.tk, "t1", 0, count)
 	// add some duplicate rows
 	for i := count - 10; i < count; i++ {
 		s.mustExec(c, "insert into t1 values (?, ?, ?)", i+10, i, i)
@@ -1330,9 +1340,7 @@ func (s *testDBSuite) testAddColumn(c *C) {
 
 	num := defaultBatchSize + 10
 	// add some rows
-	for i := 0; i < num; i++ {
-		s.mustExec(c, "insert into t2 values (?, ?, ?)", i, i, i)
-	}
+	batchInsert(s.tk, "t2", 0, num)
 
 	testddlutil.SessionExecInGoroutine(c, s.store, "alter table t2 add column c4 int default -1", done)
 
@@ -1519,15 +1527,22 @@ func (s *testDBSuite2) TestDropColumn(c *C) {
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.tk.MustExec("create database drop_col_db")
 	s.tk.MustExec("use drop_col_db")
-	s.tk.MustExec("create table t2 (c1 int, c2 int, c3 int)")
-	num := 50
+	num := 25
+	multiDDL := make([]string, 0, num)
+	sql := "create table t2 (c1 int, c2 int, c3 int, "
+	for i := 4; i < 4+num; i++ {
+		multiDDL = append(multiDDL, fmt.Sprintf("alter table t2 drop column c%d", i))
+
+		if i != 3+num {
+			sql += fmt.Sprintf("c%d int, ", i)
+		} else {
+			sql += fmt.Sprintf("c%d int)", i)
+		}
+	}
+	s.tk.MustExec(sql)
 	dmlDone := make(chan error, num)
 	ddlDone := make(chan error, num)
 
-	multiDDL := make([]string, 0, num)
-	for i := 0; i < num/2; i++ {
-		multiDDL = append(multiDDL, "alter table t2 add column c4 int", "alter table t2 drop column c4")
-	}
 	testddlutil.ExecMultiSQLInGoroutine(c, s.store, "drop_col_db", multiDDL, ddlDone)
 	for i := 0; i < num; i++ {
 		testddlutil.ExecMultiSQLInGoroutine(c, s.store, "drop_col_db", []string{"insert into t2 set c1 = 1, c2 = 1, c3 = 1, c4 = 1"}, dmlDone)
@@ -2465,16 +2480,17 @@ func (s *testDBSuite2) TestAddNotNullColumnWhileInsertOnDupUpdate(c *C) {
 				return
 			default:
 			}
-			_, tk2Err = tk2.Exec("insert nn (a, b) values (1, 1) on duplicate key update a = 1, b = b + 1")
+			_, tk2Err = tk2.Exec("insert nn (a, b) values (1, 1) on duplicate key update a = 1, b = values(b) + 1")
 			if tk2Err != nil {
 				return
 			}
 		}
 	}()
-	tk1.MustExec("alter table nn add column c int not null default 0")
+	tk1.MustExec("alter table nn add column c int not null default 3 after a")
 	close(closeCh)
 	wg.Wait()
 	c.Assert(tk2Err, IsNil)
+	tk1.MustQuery("select * from nn").Check(testkit.Rows("1 3 2"))
 }
 
 func (s *testDBSuite3) TestColumnModifyingDefinition(c *C) {

ddl/failtest/fail_db_test.go

@@ -325,6 +325,17 @@ func (s *testFailDBSuite) TestGenGlobalIDFail(c *C) {
 	tk.MustExec("admin check table t2")
 }
 
+func batchInsert(tk *testkit.TestKit, tbl string, start, end int) {
+	dml := fmt.Sprintf("insert into %s values", tbl)
+	for i := start; i < end; i++ {
+		dml += fmt.Sprintf("(%d, %d, %d)", i, i, i)
+		if i != end-1 {
+			dml += ","
+		}
+	}
+	tk.MustExec(dml)
+}
+
 func (s *testFailDBSuite) TestAddIndexWorkerNum(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("create database if not exists test_db")
@@ -334,11 +345,9 @@ func (s *testFailDBSuite) TestAddIndexWorkerNum(c *C) {
 
 	done := make(chan error, 1)
 	start := -10
-	num := 4096
 	// first add some rows
-	for i := start; i < num; i++ {
-		sql := fmt.Sprintf("insert into test_add_index values (%d, %d, %d)", i, i, i)
-		tk.MustExec(sql)
+	for i := start; i < 4090; i += 100 {
+		batchInsert(tk, "test_add_index", i, i+100)
 	}
 
 	is := s.dom.InfoSchema()

ddl/index.go

@@ -869,7 +869,7 @@ func (w *addIndexWorker) backfillIndexInTxn(handleRange reorgIndexTask) (taskCtx
 
 			// Lock the row key to notify us that someone delete or update the row,
 			// then we should not backfill the index of it, otherwise the adding index is redundant.
-			err := txn.LockKeys(context.Background(), 0, idxRecord.key)
+			err := txn.LockKeys(context.Background(), nil, 0, idxRecord.key)
 			if err != nil {
 				return errors.Trace(err)
 			}

domain/domain.go

@@ -421,7 +421,7 @@ func (do *Domain) topNSlowQueryLoop() {
 func (do *Domain) infoSyncerKeeper() {
 	defer do.wg.Done()
 	defer recoverInDomain("infoSyncerKeeper", false)
-	ticker := time.NewTicker(time.Second * time.Duration(infosync.InfoSessionTTL) / 2)
+	ticker := time.NewTicker(infosync.ReportInterval)
 	defer ticker.Stop()
 	for {
 		select {

domain/infosync/info.go

@@ -43,14 +43,15 @@ const (
 	// ServerMinStartTSPath store the server min start timestamp.
 	ServerMinStartTSPath = "/tidb/server/minstartts"
 	// keyOpDefaultRetryCnt is the default retry count for etcd store.
-	keyOpDefaultRetryCnt = 2
+	keyOpDefaultRetryCnt = 5
 	// keyOpDefaultTimeout is the default time out for etcd store.
 	keyOpDefaultTimeout = 1 * time.Second
+	// InfoSessionTTL is the ETCD session's TTL in seconds.
+	InfoSessionTTL = 10 * 60
+	// ReportInterval is interval of infoSyncerKeeper reporting min startTS.
+	ReportInterval = 30 * time.Second
 )
 
-// InfoSessionTTL is the etcd session's TTL in seconds. It's exported for testing.
-var InfoSessionTTL = 1 * 60
-
 // InfoSyncer stores server info to etcd when the tidb-server starts and delete when tidb-server shuts down.
 type InfoSyncer struct {
 	etcdCli        *clientv3.Client

executor/adapter.go

@@ -557,7 +557,7 @@ func (a *ExecStmt) handlePessimisticDML(ctx context.Context, e Executor) error {
 			return nil
 		}
 		forUpdateTS := txnCtx.GetForUpdateTS()
-		err = txn.LockKeys(ctx, forUpdateTS, keys...)
+		err = txn.LockKeys(ctx, &sctx.GetSessionVars().Killed, forUpdateTS, keys...)
 		if err == nil {
 			return nil
 		}

executor/adapter_test.go

@@ -20,7 +20,7 @@ import (
 	"github.com/pingcap/tidb/util/testkit"
 )
 
-func (s *testSuiteP1) TestQueryTime(c *C) {
+func (s *testSuiteP2) TestQueryTime(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 

executor/admin.go

@@ -432,7 +432,7 @@ func (e *RecoverIndexExec) backfillIndexInTxn(ctx context.Context, txn kv.Transa
 		}
 
 		recordKey := e.table.RecordKey(row.handle)
-		err := txn.LockKeys(ctx, 0, recordKey)
+		err := txn.LockKeys(ctx, nil, 0, recordKey)
 		if err != nil {
 			return result, err
 		}

executor/admin_test.go

@@ -44,7 +44,7 @@ func (s *testSuite1) TestAdminCheckIndexRange(c *C) {
 	result.Check(testkit.Rows("-1 hi 4", "2 cd 2"))
 }
 
-func (s *testSuite2) TestAdminRecoverIndex(c *C) {
+func (s *testSuite5) TestAdminRecoverIndex(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -158,7 +158,7 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminRecoverIndex1(c *C) {
+func (s *testSuite5) TestAdminRecoverIndex1(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	s.ctx = mock.NewContext()
 	s.ctx.Store = s.store
@@ -209,7 +209,7 @@ func (s *testSuite2) TestAdminRecoverIndex1(c *C) {
 	tk.MustExec("admin check index admin_test `primary`")
 }
 
-func (s *testSuite2) TestAdminCleanupIndex(c *C) {
+func (s *testSuite5) TestAdminCleanupIndex(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -288,7 +288,7 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) {
+func (s *testSuite5) TestAdminCleanupIndexPKNotHandle(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -336,7 +336,7 @@ func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
+func (s *testSuite5) TestAdminCleanupIndexMore(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -399,7 +399,7 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminCheckTableFailed(c *C) {
+func (s *testSuite5) TestAdminCheckTableFailed(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -505,7 +505,7 @@ func (s *testSuite2) TestAdminCheckTableFailed(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite1) TestAdminCheckTable(c *C) {
+func (s *testSuite2) TestAdminCheckTable(c *C) {
 	// test NULL value.
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
@@ -631,7 +631,7 @@ func (s *testSuite1) TestAdminCheckPrimaryIndex(c *C) {
 	tk.MustExec("admin check index t idx;")
 }
 
-func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) {
+func (s *testSuite5) TestAdminCheckWithSnapshot(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_t_s")