Support table filter (black-white-list) (pingcap#84)

* vendor: added pingcap/tidb-enterprise-tools/pkg/filter

* mydumper: accept black-white-list filtering

go.mod

@@ -14,7 +14,9 @@ require (
 	github.com/joho/sqltocsv v0.0.0-20180904231936-b24deec2b806
 	github.com/pingcap/check v0.0.0-20171206051426-1c287c953996
 	github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26
+	github.com/pingcap/parser v0.0.0-20181113072426-4a9a1b13b591
 	github.com/pingcap/tidb v0.0.0-20181120082053-012cb6da9443
+	github.com/pingcap/tidb-enterprise-tools v1.0.1-0.20181116033341-5832f7307d74
 	github.com/pkg/errors v0.8.0
 	github.com/prometheus/client_golang v0.9.0
 	github.com/satori/go.uuid v1.2.0

lightning/config/config.go

@@ -8,8 +8,9 @@ import (
 	"runtime"
 
 	"github.com/BurntSushi/toml"
-	"github.com/pkg/errors"
+	"github.com/pingcap/tidb-enterprise-tools/pkg/filter"
 	"github.com/pingcap/tidb-lightning/lightning/common"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -41,6 +42,7 @@ type Config struct {
 	// ProgressStore DBStore `toml:"progress-store" json:"progress-store"`
 	Checkpoint   Checkpoint      `toml:"checkpoint" json:"checkpoint"`
 	Mydumper     MydumperRuntime `toml:"mydumper" json:"mydumper"`
+	BWList       *filter.Rules   `toml:"black-white-list" json:"black-white-list"`
 	TikvImporter TikvImporter    `toml:"tikv-importer" json:"tikv-importer"`
 	PostRestore  PostRestore     `toml:"post-restore" json:"post-restore"`
 

lightning/mydump/loader.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/pingcap/tidb-enterprise-tools/pkg/filter"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
 	"github.com/pkg/errors"
@@ -62,13 +63,15 @@ type MDLoader struct {
 	dir      string
 	noSchema bool
 	dbs      map[string]*MDDatabaseMeta
+	filter   *filter.Filter
 }
 
 func NewMyDumpLoader(cfg *config.Config) (*MDLoader, error) {
 	mdl := &MDLoader{
 		dir:      cfg.Mydumper.SourceDir,
 		noSchema: cfg.Mydumper.NoSchema,
 		dbs:      make(map[string]*MDDatabaseMeta),
+		filter:   filter.New(true, cfg.BWList),
 	}
 
 	if err := mdl.setup(mdl.dir); err != nil {
@@ -110,6 +113,10 @@ func (l *MDLoader) setup(dir string) error {
 	return l.setupTablesData(files)
 }
 
+func (l *MDLoader) shouldSkip(table *filter.Table) bool {
+	return len(l.filter.ApplyOn([]*filter.Table{table})) == 0
+}
+
 func (l *MDLoader) setupDBs(files map[string]string) error {
 	for fpath, fname := range files {
 		if !strings.HasSuffix(fname, "-schema-create.sql") {
@@ -118,6 +125,11 @@ func (l *MDLoader) setupDBs(files map[string]string) error {
 
 		idx := strings.Index(fname, "-schema-create.sql")
 		dbname := fname[:idx]
+		if l.shouldSkip(&filter.Table{Schema: dbname}) {
+			common.AppLogger.Infof("ignoring schema file %s", fname)
+			continue
+		}
+
 		l.dbs[dbname] = &MDDatabaseMeta{
 			Name:       dbname,
 			SchemaFile: fpath,
@@ -148,6 +160,11 @@ func (l *MDLoader) setupTables(files map[string]string) error {
 		}
 
 		db, table := fields[0], fields[1]
+		if l.shouldSkip(&filter.Table{Schema: db, Name: table}) {
+			common.AppLogger.Infof("ignoring table file %s", fname)
+			continue
+		}
+
 		dbMeta, ok := l.dbs[db]
 		if !ok {
 			return errors.Errorf("invalid table schema file, cannot find db - %s", fpath)
@@ -193,6 +210,11 @@ func (l *MDLoader) setupTablesData(files map[string]string) error {
 		}
 
 		db, table := fields[0], fields[1]
+		if l.shouldSkip(&filter.Table{Schema: db, Name: table}) {
+			common.AppLogger.Infof("ignoring data file %s", fname)
+			continue
+		}
+
 		dbMeta, ok := l.dbs[db]
 		if !ok {
 			if !l.noSchema {

tests/black-white-list/data/firstdb-schema-create.sql

@@ -0,0 +1 @@
+create database firstdb;

tests/black-white-list/data/firstdb.first-schema.sql

@@ -0,0 +1 @@
+create table `first` (id int primary key);

tests/black-white-list/data/firstdb.first.1.sql

@@ -0,0 +1 @@
+insert into `first` values (8273);

tests/black-white-list/data/firstdb.first.2.sql

@@ -0,0 +1 @@
+insert into `first` values (6855);

tests/black-white-list/data/firstdb.second-schema.sql

@@ -0,0 +1 @@
+create table `second` (id int primary key);

tests/black-white-list/data/firstdb.second.1.sql

@@ -0,0 +1 @@
+create table `second` values (909);

tests/black-white-list/data/seconddb-schema-create.sql

@@ -0,0 +1 @@
+create database seconddb;

tests/black-white-list/data/seconddb.fourth-schema.sql

@@ -0,0 +1 @@
+create table `fourth` (id int primary key);

tests/black-white-list/data/seconddb.fourth.1.sql

@@ -0,0 +1 @@
+insert into `fourth` values (3256), (5457);

tests/black-white-list/data/seconddb.third-schema.sql

@@ -0,0 +1 @@
+create table `third` (id int primary key);

tests/black-white-list/data/seconddb.third.1.sql

@@ -0,0 +1 @@
+insert into `third` values (6339);

tests/black-white-list/even-table-only.toml

@@ -0,0 +1,35 @@
+[lightning]
+check-requirements = false
+file = "/tmp/lightning_test_result/lightning.log"
+level = "info"
+
+[[black-white-list.ignore-tables]]
+db-name = "firstdb"
+tbl-name = "~."
+
+[[black-white-list.do-tables]]
+db-name = "~."
+tbl-name = "second"
+
+[[black-white-list.do-tables]]
+db-name = "seconddb"
+tbl-name = "fourth"
+
+[tikv-importer]
+addr = "127.0.0.1:8808"
+
+[mydumper]
+data-source-dir = "tests/black-white-list/data"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+status-port = 10080
+pd-addr = "127.0.0.1:2379"
+log-level = "error"
+
+[post-restore]
+checksum = true
+compact = false
+analyze = false

tests/black-white-list/firstdb-only.toml

@@ -0,0 +1,26 @@
+[lightning]
+check-requirements = false
+file = "/tmp/lightning_test_result/lightning.log"
+level = "info"
+
+[black-white-list]
+do-dbs = ["~^f"]
+
+[tikv-importer]
+addr = "127.0.0.1:8808"
+
+[mydumper]
+data-source-dir = "tests/black-white-list/data"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+status-port = 10080
+pd-addr = "127.0.0.1:2379"
+log-level = "error"
+
+[post-restore]
+checksum = true
+compact = false
+analyze = false

tests/black-white-list/run.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Check if black-white-list works.
+
+set -eux
+
+run_sql 'DROP DATABASE IF EXISTS firstdb;'
+run_sql 'DROP DATABASE IF EXISTS seconddb;'
+run_lightning 'firstdb-only'
+run_sql 'SHOW DATABASES;'
+check_contains 'Database: firstdb'
+check_not_contains 'Database: seconddb'
+run_sql 'SHOW TABLES IN firstdb;'
+check_contains 'Tables_in_firstdb: first'
+check_contains 'Tables_in_firstdb: second'
+
+run_sql 'DROP DATABASE IF EXISTS firstdb;'
+run_sql 'DROP DATABASE IF EXISTS seconddb;'
+run_lightning 'even-table-only'
+run_sql 'SHOW DATABASES;'
+check_contains 'Database: firstdb'
+check_contains 'Database: seconddb'
+run_sql 'SHOW TABLES IN firstdb;'
+check_not_contains 'Tables_in_firstdb: first'
+check_contains 'Tables_in_firstdb: second'
+run_sql 'SHOW TABLES IN seconddb;'
+check_not_contains 'Tables_in_seconddb: third'
+check_contains 'Tables_in_seconddb: fourth'