Merge branch 'master' into issue_29416

cmd/ddltest/ddl_serial_test.go

@@ -33,7 +33,6 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/log"
-	zaplog "github.com/pingcap/log"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/kv"
@@ -48,7 +47,6 @@ import (
 	"github.com/pingcap/tidb/table/tables"
 	"github.com/pingcap/tidb/testkit"
 	"github.com/pingcap/tidb/types"
-	"github.com/pingcap/tidb/util/logutil"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 	goctx "golang.org/x/net/context"
@@ -91,11 +89,9 @@ type ddlSuite struct {
 }
 
 func createDDLSuite(t *testing.T) (s *ddlSuite) {
+	var err error
 	s = new(ddlSuite)
 
-	err := logutil.InitLogger(&logutil.LogConfig{Config: zaplog.Config{Level: *logLevel}})
-	require.NoError(t, err)
-
 	s.quit = make(chan struct{})
 
 	s.store, err = store.New(fmt.Sprintf("tikv://%s%s", *etcd, *tikvPath))

cmd/ddltest/main_test.go

@@ -0,0 +1,36 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ddltest
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	zaplog "github.com/pingcap/log"
+	"github.com/pingcap/tidb/util/logutil"
+	"github.com/pingcap/tidb/util/testbridge"
+	"go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+	testbridge.WorkaroundGoCheckFlags()
+	err := logutil.InitLogger(&logutil.LogConfig{Config: zaplog.Config{Level: *logLevel}})
+	if err != nil {
+		fmt.Fprint(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	goleak.VerifyTestMain(m)
+}

ddl/backfilling.go

@@ -620,6 +620,8 @@ func (w *worker) writePhysicalTableRecord(t table.PhysicalTable, bfWorkerType ba
 				backfillWorkers = append(backfillWorkers, idxWorker.backfillWorker)
 				go idxWorker.backfillWorker.run(reorgInfo.d, idxWorker, job)
 			case typeUpdateColumnWorker:
+				// Setting InCreateOrAlterStmt tells the difference between SELECT casting and ALTER COLUMN casting.
+				sessCtx.GetSessionVars().StmtCtx.InCreateOrAlterStmt = true
 				updateWorker := newUpdateColumnWorker(sessCtx, w, i, t, oldColInfo, colInfo, decodeColMap, reorgInfo.ReorgMeta.SQLMode)
 				updateWorker.priority = job.Priority
 				backfillWorkers = append(backfillWorkers, updateWorker.backfillWorker)

ddl/column_type_change_test.go

@@ -2128,6 +2128,52 @@ func (s *testColumnTypeChangeSuite) TestCastToTimeStampDecodeError(c *C) {
 	tk.MustQuery("select timestamp(cast('1000-11-11 12-3-1' as date));").Check(testkit.Rows("1000-11-11 00:00:00"))
 }
 
+// https://github.com/pingcap/tidb/issues/25285.
+func (s *testColumnTypeChangeSuite) TestCastFromZeroIntToTimeError(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test;")
+
+	prepare := func() {
+		tk.MustExec("drop table if exists t;")
+		tk.MustExec("create table t (a int);")
+		tk.MustExec("insert into t values (0);")
+	}
+	const errCodeNone = -1
+	testCases := []struct {
+		sqlMode string
+		errCode int
+	}{
+		{"STRICT_TRANS_TABLES", mysql.ErrTruncatedWrongValue},
+		{"STRICT_ALL_TABLES", mysql.ErrTruncatedWrongValue},
+		{"NO_ZERO_IN_DATE", errCodeNone},
+		{"NO_ZERO_DATE", errCodeNone},
+		{"ALLOW_INVALID_DATES", errCodeNone},
+		{"", errCodeNone},
+	}
+	for _, tc := range testCases {
+		prepare()
+		tk.MustExec(fmt.Sprintf("set @@sql_mode = '%s';", tc.sqlMode))
+		if tc.sqlMode == "NO_ZERO_DATE" {
+			tk.MustQuery(`select date(0);`).Check(testkit.Rows("<nil>"))
+		} else {
+			tk.MustQuery(`select date(0);`).Check(testkit.Rows("0000-00-00"))
+		}
+		tk.MustQuery(`select time(0);`).Check(testkit.Rows("00:00:00"))
+		if tc.errCode == errCodeNone {
+			tk.MustExec("alter table t modify column a date;")
+			prepare()
+			tk.MustExec("alter table t modify column a datetime;")
+			prepare()
+			tk.MustExec("alter table t modify column a timestamp;")
+		} else {
+			tk.MustGetErrCode("alter table t modify column a date;", mysql.ErrTruncatedWrongValue)
+			tk.MustGetErrCode("alter table t modify column a datetime;", mysql.ErrTruncatedWrongValue)
+			tk.MustGetErrCode("alter table t modify column a timestamp;", mysql.ErrTruncatedWrongValue)
+		}
+	}
+	tk.MustExec("drop table if exists t;")
+}
+
 func (s *testColumnTypeChangeSuite) TestChangeFromTimeToYear(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test;")

ddl/db_integration_test.go

@@ -1537,7 +1537,7 @@ func (s *testSerialDBSuite1) TestCreateSecondaryIndexInCluster(c *C) {
 	tk.MustExec("use test")
 
 	// test create table with non-unique key
-	tk.MustGetErrCode(`
+	tk.MustExec(`
 CREATE TABLE t (
   c01 varchar(255) NOT NULL,
   c02 varchar(255) NOT NULL,
@@ -1547,7 +1547,8 @@ CREATE TABLE t (
   c06 varchar(255) DEFAULT NULL,
   PRIMARY KEY (c01,c02,c03) clustered,
   KEY c04 (c04)
-)`, errno.ErrTooLongKey)
+)`)
+	tk.MustExec("drop table t")
 
 	// test create long clustered primary key.
 	tk.MustGetErrCode(`
@@ -1587,7 +1588,7 @@ CREATE TABLE t (
   PRIMARY KEY (c01,c02) clustered
 )`)
 	tk.MustExec("create index idx1 on t(c03)")
-	tk.MustGetErrCode("create index idx2 on t(c03, c04)", errno.ErrTooLongKey)
+	tk.MustExec("create index idx2 on t(c03, c04)")
 	tk.MustExec("create unique index uk2 on t(c03, c04)")
 	tk.MustExec("drop table t")
 
@@ -1606,9 +1607,9 @@ CREATE TABLE t (
 )`)
 	tk.MustExec("alter table t change c03 c10 varchar(256) default null")
 	tk.MustGetErrCode("alter table t change c10 c100 varchar(1024) default null", errno.ErrTooLongKey)
-	tk.MustGetErrCode("alter table t modify c10 varchar(600) default null", errno.ErrTooLongKey)
+	tk.MustExec("alter table t modify c10 varchar(600) default null")
 	tk.MustExec("alter table t modify c06 varchar(600) default null")
-	tk.MustGetErrCode("alter table t modify c01 varchar(510)", errno.ErrTooLongKey)
+	tk.MustExec("alter table t modify c01 varchar(510)")
 	tk.MustExec("create table t2 like t")
 }
 

ddl/db_test.go

@@ -5474,8 +5474,7 @@ func (s *testDBSuite1) TestModifyColumnTime_YearToDate(c *C) {
 		{"year", `"69"`, "date", "", errno.ErrTruncatedWrongValue},
 		{"year", `"70"`, "date", "", errno.ErrTruncatedWrongValue},
 		{"year", `"99"`, "date", "", errno.ErrTruncatedWrongValue},
-		// MySQL will get "Data truncation: Incorrect date value: '0000'", but TiDB treat 00 as valid datetime.
-		{"year", `00`, "date", "0000-00-00", 0},
+		{"year", `00`, "date", "", errno.ErrTruncatedWrongValue},
 		{"year", `69`, "date", "", errno.ErrTruncatedWrongValue},
 		{"year", `70`, "date", "", errno.ErrTruncatedWrongValue},
 		{"year", `99`, "date", "", errno.ErrTruncatedWrongValue},
@@ -5492,8 +5491,7 @@ func (s *testDBSuite1) TestModifyColumnTime_YearToDatetime(c *C) {
 		{"year", `"69"`, "datetime", "", errno.ErrTruncatedWrongValue},
 		{"year", `"70"`, "datetime", "", errno.ErrTruncatedWrongValue},
 		{"year", `"99"`, "datetime", "", errno.ErrTruncatedWrongValue},
-		// MySQL will get "Data truncation: Incorrect date value: '0000'", but TiDB treat 00 as valid datetime.
-		{"year", `00`, "datetime", "0000-00-00 00:00:00", 0},
+		{"year", `00`, "datetime", "", errno.ErrTruncatedWrongValue},
 		{"year", `69`, "datetime", "", errno.ErrTruncatedWrongValue},
 		{"year", `70`, "datetime", "", errno.ErrTruncatedWrongValue},
 		{"year", `99`, "datetime", "", errno.ErrTruncatedWrongValue},
@@ -5510,8 +5508,7 @@ func (s *testDBSuite1) TestModifyColumnTime_YearToTimestamp(c *C) {
 		{"year", `"69"`, "timestamp", "", errno.ErrTruncatedWrongValue},
 		{"year", `"70"`, "timestamp", "", errno.ErrTruncatedWrongValue},
 		{"year", `"99"`, "timestamp", "", errno.ErrTruncatedWrongValue},
-		// MySQL will get "Data truncation: Incorrect date value: '0000'", but TiDB treat 00 as valid datetime.
-		{"year", `00`, "timestamp", "0000-00-00 00:00:00", 0},
+		{"year", `00`, "timestamp", "", errno.ErrTruncatedWrongValue},
 		{"year", `69`, "timestamp", "", errno.ErrTruncatedWrongValue},
 		{"year", `70`, "timestamp", "", errno.ErrTruncatedWrongValue},
 		{"year", `99`, "timestamp", "", errno.ErrTruncatedWrongValue},

ddl/ddl_api.go

@@ -1631,27 +1631,7 @@ func buildTableInfo(
 		idxInfo.ID = allocateIndexID(tbInfo)
 		tbInfo.Indices = append(tbInfo.Indices, idxInfo)
 	}
-	if tbInfo.IsCommonHandle {
-		// Ensure tblInfo's each non-unique secondary-index's len + primary-key's len <= MaxIndexLength for clustered index table.
-		var pkLen, idxLen int
-		pkLen, err = indexColumnsLen(tbInfo.Columns, tables.FindPrimaryIndex(tbInfo).Columns)
-		if err != nil {
-			return
-		}
-		for _, idx := range tbInfo.Indices {
-			if idx.Unique {
-				// Only need check for non-unique secondary-index.
-				continue
-			}
-			idxLen, err = indexColumnsLen(tbInfo.Columns, idx.Columns)
-			if err != nil {
-				return
-			}
-			if pkLen+idxLen > config.GetGlobalConfig().MaxIndexLength {
-				return nil, errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength)
-			}
-		}
-	}
+
 	return
 }
 
@@ -4272,9 +4252,6 @@ func (d *ddl) getModifiableColumnJob(ctx context.Context, sctx sessionctx.Contex
 // Index has a max-prefix-length constraint. eg: a varchar(100), index idx(a), modifying column a to a varchar(4000)
 // will cause index idx to break the max-prefix-length constraint.
 //
-// For clustered index:
-//  Change column in pk need recheck all non-unique index, new pk len + index len < maxIndexLength.
-//  Change column in secondary only need related index, pk len + new index len < maxIndexLength.
 func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol *model.ColumnInfo) error {
 	columns := make([]*model.ColumnInfo, 0, len(tbInfo.Columns))
 	columns = append(columns, tbInfo.Columns...)
@@ -4289,40 +4266,31 @@ func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol
 	}
 
 	pkIndex := tables.FindPrimaryIndex(tbInfo)
-	var clusteredPkLen int
-	if tbInfo.IsCommonHandle {
-		var err error
-		clusteredPkLen, err = indexColumnsLen(columns, pkIndex.Columns)
-		if err != nil {
-			return err
-		}
-	}
 
-	checkOneIndex := func(indexInfo *model.IndexInfo, pkLenAppendToKey int, skipCheckIfNotModify bool) (modified bool, err error) {
+	checkOneIndex := func(indexInfo *model.IndexInfo) (err error) {
+		var modified bool
 		for _, col := range indexInfo.Columns {
 			if col.Name.L == originalCol.Name.L {
 				modified = true
 				break
 			}
 		}
-		if skipCheckIfNotModify && !modified {
+		if !modified {
 			return
 		}
 		err = checkIndexInModifiableColumns(columns, indexInfo.Columns)
 		if err != nil {
 			return
 		}
-		err = checkIndexPrefixLength(columns, indexInfo.Columns, pkLenAppendToKey)
+		err = checkIndexPrefixLength(columns, indexInfo.Columns)
 		return
 	}
 
-	// Check primary key first and get "does primary key's column has be modified?" info.
-	var (
-		pkModified bool
-		err        error
-	)
+	// Check primary key first.
+	var err error
+
 	if pkIndex != nil {
-		pkModified, err = checkOneIndex(pkIndex, 0, true)
+		err = checkOneIndex(pkIndex)
 		if err != nil {
 			return err
 		}
@@ -4333,12 +4301,9 @@ func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol
 		if indexInfo.Primary {
 			continue
 		}
-		var pkLenAppendToKey int
-		if !indexInfo.Unique {
-			pkLenAppendToKey = clusteredPkLen
-		}
-
-		_, err = checkOneIndex(indexInfo, pkLenAppendToKey, !tbInfo.IsCommonHandle || !pkModified)
+		// the second param should always be set to true, check index length only if it was modified
+		// checkOneIndex needs one param only.
+		err = checkOneIndex(indexInfo)
 		if err != nil {
 			return err
 		}
@@ -5481,22 +5446,6 @@ func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde
 		return errors.Trace(err)
 	}
 
-	if !unique && tblInfo.IsCommonHandle {
-		// Ensure new created non-unique secondary-index's len + primary-key's len <= MaxIndexLength in clustered index table.
-		var pkLen, idxLen int
-		pkLen, err = indexColumnsLen(tblInfo.Columns, tables.FindPrimaryIndex(tblInfo).Columns)
-		if err != nil {
-			return err
-		}
-		idxLen, err = indexColumnsLen(finalColumns, indexColumns)
-		if err != nil {
-			return err
-		}
-		if pkLen+idxLen > config.GetGlobalConfig().MaxIndexLength {
-			return errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength)
-		}
-	}
-
 	global := false
 	if unique && tblInfo.GetPartitionInfo() != nil {
 		ck, err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), indexColumns, tblInfo)

ddl/index.go

@@ -110,12 +110,12 @@ func checkPKOnGeneratedColumn(tblInfo *model.TableInfo, indexPartSpecifications
 	return lastCol, nil
 }
 
-func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn, pkLenAppendToKey int) error {
+func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn) error {
 	idxLen, err := indexColumnsLen(columns, idxColumns)
 	if err != nil {
 		return err
 	}
-	if idxLen+pkLenAppendToKey > config.GetGlobalConfig().MaxIndexLength {
+	if idxLen > config.GetGlobalConfig().MaxIndexLength {
 		return errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength)
 	}
 	return nil

distsql/request_builder.go

@@ -721,6 +721,8 @@ func encodeIndexKey(sc *stmtctx.StatementContext, ran *ranger.Range) ([]byte, []
 		}
 	}
 
+	// NOTE: this is a hard-code operation to avoid wrong results when accessing unique index with NULL;
+	// Please see https://github.com/pingcap/tidb/issues/29650 for more details
 	if hasNull {
 		// Append 0 to make unique-key range [null, null] to be a scan rather than point-get.
 		high = kv.Key(high).Next()

executor/adapter.go

@@ -1014,6 +1014,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
 		ResultRows:        GetResultRowsCount(a.Ctx, a.Plan),
 		ExecRetryCount:    a.retryCount,
 		IsExplicitTxn:     sessVars.TxnCtx.IsExplicit,
+		IsWriteCacheTable: sessVars.StmtCtx.WaitLockLeaseTime > 0,
 	}
 	if a.retryCount > 0 {
 		slowItems.ExecRetryTime = costTime - sessVars.DurationParse - sessVars.DurationCompile - time.Since(a.retryStartTime)
@@ -1193,6 +1194,9 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
 	if tikvExecDetailRaw != nil {
 		tikvExecDetail = *(tikvExecDetailRaw.(*util.ExecDetails))
 	}
+	if stmtCtx.WaitLockLeaseTime > 0 {
+		execDetail.BackoffSleep["waitLockLeaseForCacheTable"] = stmtCtx.WaitLockLeaseTime
+	}
 	stmtExecInfo := &stmtsummary.StmtExecInfo{
 		SchemaName:      strings.ToLower(sessVars.CurrentDB),
 		OriginalSQL:     sql,

executor/executor.go

@@ -983,7 +983,12 @@ func newLockCtx(seVars *variable.SessionVars, lockWaitTime int64) *tikvstore.Loc
 		}
 		if mutation := req.Mutations[0]; mutation != nil {
 			label := resourcegrouptag.GetResourceGroupLabelByKey(mutation.Key)
-			return seVars.StmtCtx.GetResourceGroupTagByLabel(label)
+			normalized, digest := seVars.StmtCtx.SQLDigest()
+			if len(normalized) == 0 {
+				return nil
+			}
+			_, planDigest := seVars.StmtCtx.GetPlanDigest()
+			return resourcegrouptag.EncodeResourceGroupTag(digest, planDigest, label)
 		}
 		return nil
 	}