config,mydumper: replace black-white-list by table-filter

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/DATA-DOG/go-sqlmock v1.4.1
+	github.com/carlmjohnson/flagext v0.0.11
 	github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b
 	github.com/coreos/go-semver v0.3.0
 	github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 // indirect
@@ -23,7 +24,7 @@ require (
 	github.com/pingcap/parser v0.0.0-20200522094936-3b720a0512a6
 	github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181
 	github.com/pingcap/tidb v1.1.0-beta.0.20200527030457-572bba0499e1
-	github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible
+	github.com/pingcap/tidb-tools v4.0.1+incompatible
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/satori/go.uuid v1.2.0

go.sum

@@ -53,6 +53,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ=
 github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU=
+github.com/carlmjohnson/flagext v0.0.11 h1:k2K/FCzmfdoYsDLZpHw8w/+V2Rt5gRgCXEfR4Y3d2K4=
+github.com/carlmjohnson/flagext v0.0.11/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac=
 github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
@@ -143,6 +145,7 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -280,6 +283,7 @@ github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZO
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hypnoglow/gormzap v0.3.0/go.mod h1:5Wom8B7Jl2oK0Im9hs6KQ+Kl92w4Y7gKCrj66rhyvw0=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -310,6 +314,7 @@ github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJh
 github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok=
 github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9 h1:Y+lzErDTURqeXqlqYi4YBYbDd7ycU74gW1ADt57/bgY=
 github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJm034bVoG30IshVm688T2hi8=
 github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
@@ -401,13 +406,16 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y=
 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
+github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
@@ -481,6 +489,8 @@ github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incomp
 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible h1:/JKsYjsa5Ug8v5CN4zIbJGIqsvgBUkGwaP/rEScVvWM=
 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
+github.com/pingcap/tidb-tools v4.0.1+incompatible h1:/Z0FEc0Os4CZ0Wun7uyg9teEwcHIhNz1iwGTIC86uhg=
+github.com/pingcap/tidb-tools v4.0.1+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee h1:XJQ6/LGzOSc/jo33AD8t7jtc4GohxcyODsYnb+kZXJM=
 github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
@@ -885,11 +895,13 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
 gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
+gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4 h1:hILp2hNrRnYjZpmIbx70psAHbBSEcQ1NIzDcUbJ1b6g=
 gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
@@ -906,6 +918,7 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

lightning/config/config.go

@@ -28,7 +28,7 @@ import (
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/log"
-	"github.com/pingcap/tidb-tools/pkg/filter"
+	"github.com/pingcap/tidb-tools/pkg/table-filter"
 	router "github.com/pingcap/tidb-tools/pkg/table-router"
 	tidbcfg "github.com/pingcap/tidb/config"
 	"go.uber.org/zap"
@@ -91,12 +91,13 @@ type Config struct {
 
 	Checkpoint   Checkpoint          `toml:"checkpoint" json:"checkpoint"`
 	Mydumper     MydumperRuntime     `toml:"mydumper" json:"mydumper"`
-	BWList       *filter.Rules       `toml:"black-white-list" json:"black-white-list"`
 	TikvImporter TikvImporter        `toml:"tikv-importer" json:"tikv-importer"`
 	PostRestore  PostRestore         `toml:"post-restore" json:"post-restore"`
 	Cron         Cron                `toml:"cron" json:"cron"`
 	Routes       []*router.TableRule `toml:"routes" json:"routes"`
 	Security     Security            `toml:"security" json:"security"`
+
+	BWList filter.MySQLReplicationRules `toml:"black-white-list" json:"black-white-list"`
 }
 
 func (c *Config) String() string {
@@ -149,6 +150,7 @@ type MydumperRuntime struct {
 	CaseSensitive    bool      `toml:"case-sensitive" json:"case-sensitive"`
 	StrictFormat     bool      `toml:"strict-format" json:"strict-format"`
 	MaxRegionSize    int64     `toml:"max-region-size" json:"max-region-size"`
+	Filter           []string  `toml:"filter" json:"filter"`
 }
 
 type TikvImporter struct {
@@ -256,6 +258,7 @@ func NewConfig() *Config {
 			},
 			StrictFormat:  false,
 			MaxRegionSize: MaxRegionSize,
+			Filter:        []string{"*.*"},
 		},
 		TikvImporter: TikvImporter{
 			Backend:         BackendImporter,
@@ -268,7 +271,6 @@ func NewConfig() *Config {
 			Checksum: true,
 			Analyze:  true,
 		},
-		BWList: &filter.Rules{},
 	}
 }
 
@@ -286,6 +288,7 @@ func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error {
 	cfg.TiDB.PdAddr = global.TiDB.PdAddr
 	cfg.Mydumper.SourceDir = global.Mydumper.SourceDir
 	cfg.Mydumper.NoSchema = global.Mydumper.NoSchema
+	cfg.Mydumper.Filter = global.Mydumper.Filter
 	cfg.TikvImporter.Addr = global.TikvImporter.Addr
 	cfg.TikvImporter.Backend = global.TikvImporter.Backend
 	cfg.TikvImporter.SortedKVDir = global.TikvImporter.SortedKVDir
@@ -454,12 +457,13 @@ func (cfg *Config) Adjust() error {
 		return errors.Errorf("invalid config: unsupported `tidb.tls` config %s", cfg.TiDB.TLS)
 	}
 
-	cfg.BWList.IgnoreDBs = append(cfg.BWList.IgnoreDBs,
-		"mysql",
-		"information_schema",
-		"performance_schema",
-		"sys",
-	)
+	// mydumper.filter and black-white-list cannot co-exist.
+	if cfg.HasLegacyBlackWhiteList() {
+		log.L().Warn("the config `black-white-list` has been deprecated, please replace with `mydumper.filter`")
+		if !(len(cfg.Mydumper.Filter) == 1 && cfg.Mydumper.Filter[0] == "*.*") {
+			return errors.New("invalid config: `mydumper.filter` and `black-white-list` cannot be simultaneously defined")
+		}
+	}
 
 	for _, rule := range cfg.Routes {
 		if !cfg.Mydumper.CaseSensitive {
@@ -538,3 +542,9 @@ func (cfg *Config) Adjust() error {
 
 	return nil
 }
+
+// HasLegacyBlackWhiteList checks whether the deprecated [black-white-list] section
+// was defined.
+func (cfg *Config) HasLegacyBlackWhiteList() bool {
+	return len(cfg.BWList.DoTables) != 0 || len(cfg.BWList.DoDBs) != 0 || len(cfg.BWList.IgnoreTables) != 0 || len(cfg.BWList.IgnoreDBs) != 0
+}

lightning/config/global.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
+	"github.com/carlmjohnson/flagext"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/log"
@@ -49,8 +50,9 @@ type GlobalTiDB struct {
 }
 
 type GlobalMydumper struct {
-	SourceDir string `toml:"data-source-dir" json:"data-source-dir"`
-	NoSchema  bool   `toml:"no-schema" json:"no-schema"`
+	SourceDir string   `toml:"data-source-dir" json:"data-source-dir"`
+	NoSchema  bool     `toml:"no-schema" json:"no-schema"`
+	Filter    []string `toml:"filter" json:"filter"`
 }
 
 type GlobalImporter struct {
@@ -95,6 +97,9 @@ func NewGlobalConfig() *GlobalConfig {
 			StatusPort: 10080,
 			LogLevel:   "error",
 		},
+		Mydumper: GlobalMydumper{
+			Filter: []string{"*.*"},
+		},
 		TikvImporter: GlobalImporter{
 			Backend: "importer",
 		},
@@ -136,7 +141,7 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon
 	fs.StringVar(&configFilePath, "config", "", "tidb-lightning configuration file")
 	printVersion := fs.Bool("V", false, "print version of lightning")
 
-	logLevel := fs.String("L", "", `log level: info, debug, warn, error, fatal (default "info")`)
+	logLevel := flagext.ChoiceVar(fs, "L", "", `log level: info, debug, warn, error, fatal (default info)`, "", "info", "debug", "warn", "warning", "error", "fatal")
 	logFilePath := fs.String("log-file", timestampLogFileName(), "log file path")
 	tidbHost := fs.String("tidb-host", "", "TiDB server host")
 	tidbPort := fs.Int("tidb-port", 0, "TiDB server port (default 4000)")
@@ -146,7 +151,7 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon
 	pdAddr := fs.String("pd-urls", "", "PD endpoint address")
 	dataSrcPath := fs.String("d", "", "Directory of the dump to import")
 	importerAddr := fs.String("importer", "", "address (host:port) to connect to tikv-importer")
-	backend := fs.String("backend", "", `delivery backend ("importer" or "tidb" or "local")`)
+	backend := flagext.ChoiceVar(fs, "backend", "", `delivery backend: importer, tidb, local (default importer)`, "", "importer", "tidb", "local")
 	sortedKVDir := fs.String("sorted-kv-dir", "", "path for KV pairs when local backend enabled")
 	enableCheckpoint := fs.Bool("enable-checkpoint", true, "whether to enable checkpoints")
 	noSchema := fs.Bool("no-schema", false, "ignore schema files, get schema directly from TiDB instead")
@@ -160,6 +165,9 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon
 	statusAddr := fs.String("status-addr", "", "the Lightning server address")
 	serverMode := fs.Bool("server-mode", false, "start Lightning in server mode, wait for multiple tasks instead of starting immediately")
 
+	var filter []string
+	flagext.StringsVar(fs, &filter, "f", "select tables to import")
+
 	if extraFlags != nil {
 		extraFlags(fs)
 	}
@@ -252,6 +260,9 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon
 	if *tlsKeyPath != "" {
 		cfg.Security.KeyPath = *tlsKeyPath
 	}
+	if len(filter) > 0 {
+		cfg.Mydumper.Filter = filter
+	}
 
 	if cfg.App.StatusAddr == "" && cfg.App.ServerMode {
 		return nil, errors.New("If server-mode is enabled, the status-addr must be a valid listen address")

lightning/lightning_test.go

@@ -58,7 +58,10 @@ func (s *lightningSuite) TestRun(c *C) {
 	c.Assert(err, ErrorMatches, ".*mydumper dir does not exist")
 
 	err = lightning.run(&config.Config{
-		Mydumper: config.MydumperRuntime{SourceDir: "."},
+		Mydumper: config.MydumperRuntime{
+			SourceDir: ".",
+			Filter:    []string{"*.*"},
+		},
 		Checkpoint: config.Checkpoint{
 			Enable: true,
 			Driver: "invalid",
@@ -67,7 +70,10 @@ func (s *lightningSuite) TestRun(c *C) {
 	c.Assert(err, ErrorMatches, "Unknown checkpoint driver invalid")
 
 	err = lightning.run(&config.Config{
-		Mydumper: config.MydumperRuntime{SourceDir: "."},
+		Mydumper: config.MydumperRuntime{
+			SourceDir: ".",
+			Filter:    []string{"*.*"},
+		},
 		Checkpoint: config.Checkpoint{
 			Enable: true,
 			Driver: "file",

lightning/mydump/loader.go

@@ -24,7 +24,7 @@ import (
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
 	"github.com/pingcap/tidb-lightning/lightning/log"
-	"github.com/pingcap/tidb-tools/pkg/filter"
+	"github.com/pingcap/tidb-tools/pkg/table-filter"
 	router "github.com/pingcap/tidb-tools/pkg/table-router"
 	"go.uber.org/zap"
 )
@@ -64,7 +64,7 @@ type MDLoader struct {
 	dir      string
 	noSchema bool
 	dbs      []*MDDatabaseMeta
-	filter   *filter.Filter
+	filter   filter.Filter
 	router   *router.Table
 	charSet  string
 }
@@ -80,18 +80,27 @@ type mdLoaderSetup struct {
 
 func NewMyDumpLoader(cfg *config.Config) (*MDLoader, error) {
 	var r *router.Table
+	var err error
 	if len(cfg.Routes) > 0 {
-		var err error
 		r, err = router.NewTableRouter(cfg.Mydumper.CaseSensitive, cfg.Routes)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
 	}
 
-	f, err := filter.New(false, cfg.BWList)
+	// use the legacy black-white-list if defined. otherwise use the new filter.
+	var f filter.Filter
+	if cfg.HasLegacyBlackWhiteList() {
+		f, err = filter.ParseMySQLReplicationRules(&cfg.BWList)
+	} else {
+		f, err = filter.Parse(cfg.Mydumper.Filter)
+	}
 	if err != nil {
 		return nil, err
 	}
+	if !cfg.Mydumper.CaseSensitive {
+		f = filter.CaseInsensitive(f)
+	}
 
 	mdl := &MDLoader{
 		dir:      cfg.Mydumper.SourceDir,
@@ -297,7 +306,10 @@ func (s *mdLoaderSetup) listFiles(dir string) error {
 }
 
 func (l *MDLoader) shouldSkip(table *filter.Table) bool {
-	return len(l.filter.ApplyOn([]*filter.Table{table})) == 0
+	if len(table.Name) == 0 {
+		return !l.filter.MatchSchema(table.Schema)
+	}
+	return !l.filter.MatchTable(table.Schema, table.Name)
 }
 
 func (s *mdLoaderSetup) route() error {

lightning/mydump/loader_test.go

@@ -38,8 +38,17 @@ type testMydumpLoaderSuite struct {
 func (s *testMydumpLoaderSuite) SetUpSuite(c *C)    {}
 func (s *testMydumpLoaderSuite) TearDownSuite(c *C) {}
 
+func newConfigWithSourceDir(sourceDir string) *config.Config {
+	return &config.Config{
+		Mydumper: config.MydumperRuntime{
+			SourceDir: sourceDir,
+			Filter:    []string{"*.*"},
+		},
+	}
+}
+
 func (s *testMydumpLoaderSuite) SetUpTest(c *C) {
-	s.cfg = &config.Config{Mydumper: config.MydumperRuntime{SourceDir: c.MkDir()}}
+	s.cfg = newConfigWithSourceDir(c.MkDir())
 }
 
 func (s *testMydumpLoaderSuite) touch(c *C, filename ...string) string {
@@ -59,11 +68,11 @@ func (s *testMydumpLoaderSuite) mkdir(c *C, dirname string) {
 }
 
 func (s *testMydumpLoaderSuite) TestLoader(c *C) {
-	cfg := &config.Config{Mydumper: config.MydumperRuntime{SourceDir: "./not-exists"}}
+	cfg := newConfigWithSourceDir("./not-exists")
 	_, err := md.NewMyDumpLoader(cfg)
 	c.Assert(err, NotNil)
 
-	cfg = &config.Config{Mydumper: config.MydumperRuntime{SourceDir: "./examples"}}
+	cfg = newConfigWithSourceDir("./examples")
 	mdl, err := md.NewMyDumpLoader(cfg)
 	c.Assert(err, IsNil)
 

lightning/mydump/region_test.go

@@ -59,7 +59,7 @@ func getFileSize(file string) (int64, error) {
 	TODO : test with specified 'regionBlockSize' ...
 */
 func (s *testMydumpRegionSuite) TestTableRegion(c *C) {
-	cfg := &config.Config{Mydumper: config.MydumperRuntime{SourceDir: "./examples"}}
+	cfg := newConfigWithSourceDir("./examples")
 	loader, _ := NewMyDumpLoader(cfg)
 	dbMeta := loader.GetDatabases()[0]
 
@@ -216,6 +216,7 @@ func (s *testMydumpRegionSuite) TestSplitLargeFile(c *C) {
 				BackslashEscape: true,
 			},
 			StrictFormat: true,
+			Filter:       []string{"*.*"},
 		},
 	}
 	filePath := "./csv/split_large_file.csv"