kv: fix handling of column default values (pingcap#170)

* kv: fix handling of column default values

* if the column is AUTO_INCREMENT, fill in with row_id (assume it is
  missing for the entire table instead of just a few values)
* if the column has DEFAULT, fill in that value

* tests: ensure DEFAULT CURRENT_TIMESTAMP works

lightning/kv/sql2kv.go

@@ -69,13 +69,16 @@ func (kvcodec *TableKVEncoder) Encode(
 	}
 
 	for i, col := range cols {
-		if j := columnPermutation[i]; j >= 0 {
+		if j := columnPermutation[i]; j >= 0 && j < len(row) {
 			value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo())
 			if err == nil {
 				value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx)
 			}
+		} else if mysql.HasAutoIncrementFlag(col.Flag) {
+			// we still need a conversion, e.g. to catch overflow with a TINYINT column.
+			value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo())
 		} else {
-			value, err = table.GetColOriginDefaultValue(kvcodec.se, col.ToInfo())
+			value, err = table.GetColDefaultValue(kvcodec.se, col.ToInfo())
 		}
 		if err != nil {
 			return nil, errors.Trace(err)

tests/default-columns/config.toml

@@ -0,0 +1,18 @@
+[lightning]
+check-requirements = false
+file = "/tmp/lightning_test_result/lightning.log"
+level = "debug"
+
+[tikv-importer]
+addr = "127.0.0.1:8808"
+
+[mydumper]
+data-source-dir = "tests/default-columns/data"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+status-port = 10080
+pd-addr = "127.0.0.1:2379"
+log-level = "error"

tests/default-columns/data/defcol-schema-create.sql

@@ -0,0 +1 @@
+CREATE DATABASE defcol;

tests/default-columns/data/defcol.t-schema.sql

@@ -0,0 +1,6 @@
+CREATE TABLE t (
+    pk INT PRIMARY KEY AUTO_INCREMENT,
+    x INT NULL,
+    y INT NOT NULL DEFAULT 123,
+    z DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);

tests/default-columns/data/defcol.t.1.sql

@@ -0,0 +1 @@
+INSERT INTO t () VALUES (), (), (), (), (), ();

tests/default-columns/data/defcol.t.2.sql

@@ -0,0 +1 @@
+INSERT INTO t VALUES (), (), ();

tests/default-columns/data/defcol.u-schema.sql

@@ -0,0 +1,4 @@
+CREATE TABLE u (
+    xx INT UNIQUE AUTO_INCREMENT,
+    yy INT PRIMARY KEY
+);

tests/default-columns/data/defcol.u.1.sql

@@ -0,0 +1 @@
+INSERT INTO u (yy) VALUES (40), (60);

tests/default-columns/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+run_sql 'DROP DATABASE IF EXISTS defcol'
+
+run_lightning
+
+run_sql 'SELECT min(pk), count(pk) FROM defcol.t'
+check_contains 'min(pk): 1'
+check_contains 'count(pk): 9'
+
+run_sql 'SELECT pk FROM defcol.t WHERE x IS NOT NULL OR y <> 123 OR z IS NULL OR z NOT BETWEEN now() - INTERVAL 5 MINUTE AND now()'
+check_not_contains 'pk:'
+
+run_sql 'SELECT xx FROM defcol.u WHERE yy = 40'
+check_contains 'xx: 1'
+
+run_sql 'SELECT xx FROM defcol.u WHERE yy = 60'
+check_contains 'xx: 2'