github.com/pingcap/tidb-lightning@v5.0.0-rc.0.20210428090220-84b649866577+incompatible/lightning/mydump/loader_test.go

// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package mydump_test

import (
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	. "github.com/pingcap/check"
	filter "github.com/pingcap/tidb-tools/pkg/table-filter"
	router "github.com/pingcap/tidb-tools/pkg/table-router"

	"github.com/pingcap/tidb-lightning/lightning/config"
	md "github.com/pingcap/tidb-lightning/lightning/mydump"
)

var _ = Suite(&testMydumpLoaderSuite{})

func TestMydumps(t *testing.T) {
	TestingT(t)
}

type testMydumpLoaderSuite struct {
	cfg       *config.Config
	sourceDir string
}

func (s *testMydumpLoaderSuite) SetUpSuite(c *C)    {}
func (s *testMydumpLoaderSuite) TearDownSuite(c *C) {}

func newConfigWithSourceDir(sourceDir string) *config.Config {
	path, _ := filepath.Abs(sourceDir)
	return &config.Config{
		Mydumper: config.MydumperRuntime{
			SourceDir:        "file://" + filepath.ToSlash(path),
			Filter:           []string{"*.*"},
			DefaultFileRules: true,
		},
	}
}

func (s *testMydumpLoaderSuite) SetUpTest(c *C) {
	s.sourceDir = c.MkDir()
	s.cfg = newConfigWithSourceDir(s.sourceDir)
}

func (s *testMydumpLoaderSuite) touch(c *C, filename ...string) string {
	components := make([]string, len(filename)+1)
	components = append(components, s.sourceDir)
	components = append(components, filename...)
	path := filepath.Join(components...)
	err := ioutil.WriteFile(path, nil, 0644)
	c.Assert(err, IsNil)
	return path
}

func (s *testMydumpLoaderSuite) mkdir(c *C, dirname string) {
	path := filepath.Join(s.sourceDir, dirname)
	err := os.Mkdir(path, 0755)
	c.Assert(err, IsNil)
}

func (s *testMydumpLoaderSuite) TestLoader(c *C) {
	ctx := context.Background()
	cfg := newConfigWithSourceDir("./not-exists")
	_, err := md.NewMyDumpLoader(ctx, cfg)
	c.Assert(err, NotNil)

	cfg = newConfigWithSourceDir("./examples")
	mdl, err := md.NewMyDumpLoader(ctx, cfg)
	c.Assert(err, IsNil)

	dbMetas := mdl.GetDatabases()
	c.Assert(len(dbMetas), Equals, 1)
	dbMeta := dbMetas[0]
	c.Assert(dbMeta.Name, Equals, "mocker_test")
	c.Assert(len(dbMeta.Tables), Equals, 4)

	expected := []struct {
		name      string
		dataFiles int
	}{
		{name: "i", dataFiles: 1},
		{name: "report_case_high_risk", dataFiles: 1},
		{name: "tbl_multi_index", dataFiles: 1},
		{name: "tbl_autoid", dataFiles: 1},
	}

	for i, table := range expected {
		c.Assert(dbMeta.Tables[i].Name, Equals, table.name)
		c.Assert(len(dbMeta.Tables[i].DataFiles), Equals, table.dataFiles)
	}
}

func (s *testMydumpLoaderSuite) TestEmptyDB(c *C) {
	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, "no schema create sql files found. Please either set `mydumper.no-schema` to true or add schema sql file for each database.")
}

func (s *testMydumpLoaderSuite) TestDuplicatedDB(c *C) {
	/*
		Path/
			a/
				db-schema-create.sql
			b/
				db-schema-create.sql
	*/
	s.mkdir(c, "a")
	s.touch(c, "a", "db-schema-create.sql")
	s.mkdir(c, "b")
	s.touch(c, "b", "db-schema-create.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid database schema file, duplicated item - .*[/\\]db-schema-create\.sql`)
}

func (s *testMydumpLoaderSuite) TestTableNoHostDB(c *C) {
	/*
		Path/
			notdb-schema-create.sql
			db.tbl-schema.sql
	*/

	dir := s.sourceDir
	err := ioutil.WriteFile(filepath.Join(dir, "notdb-schema-create.sql"), nil, 0644)
	c.Assert(err, IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "db.tbl-schema.sql"), nil, 0644)
	c.Assert(err, IsNil)

	_, err = md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid table schema file, cannot find db 'db' - .*db\.tbl-schema\.sql`)
}

func (s *testMydumpLoaderSuite) TestDuplicatedTable(c *C) {
	/*
		Path/
			db-schema-create.sql
			a/
				db.tbl-schema.sql
			b/
				db.tbl-schema.sql
	*/

	s.touch(c, "db-schema-create.sql")
	s.mkdir(c, "a")
	s.touch(c, "a", "db.tbl-schema.sql")
	s.mkdir(c, "b")
	s.touch(c, "b", "db.tbl-schema.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid table schema file, duplicated item - .*db\.tbl-schema\.sql`)
}

func (s *testMydumpLoaderSuite) TestDataNoHostDB(c *C) {
	/*
		Path/
			notdb-schema-create.sql
			db.tbl.sql
	*/

	s.touch(c, "notdb-schema-create.sql")
	s.touch(c, "db.tbl.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid data file, miss host db 'db' - .*[/\\]?db\.tbl\.sql`)
}

func (s *testMydumpLoaderSuite) TestDataNoHostTable(c *C) {
	/*
		Path/
			db-schema-create.sql
			db.tbl.sql
	*/

	s.touch(c, "db-schema-create.sql")
	s.touch(c, "db.tbl.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid data file, miss host table 'tbl' - .*[/\\]?db\.tbl\.sql`)
}

func (s *testMydumpLoaderSuite) TestViewNoHostDB(c *C) {
	/*
		Path/
			notdb-schema-create.sql
			db.tbl-schema-view.sql
	*/
	s.touch(c, "notdb-schema-create.sql")
	s.touch(c, "db.tbl-schema-view.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid table schema file, cannot find db 'db' - .*[/\\]?db\.tbl-schema-view\.sql`)
}

func (s *testMydumpLoaderSuite) TestViewNoHostTable(c *C) {
	/*
		Path/
			db-schema-create.sql
			db.tbl-schema-view.sql
	*/

	s.touch(c, "db-schema-create.sql")
	s.touch(c, "db.tbl-schema-view.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `invalid view schema file, miss host table schema for view 'tbl'`)
}

func (s *testMydumpLoaderSuite) TestDataWithoutSchema(c *C) {
	dir := s.sourceDir
	p := filepath.Join(dir, "db.tbl.sql")
	err := ioutil.WriteFile(p, nil, 0644)
	c.Assert(err, IsNil)

	s.cfg.Mydumper.NoSchema = true

	mdl, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, IsNil)
	c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{{
		Name:       "db",
		SchemaFile: "",
		Tables: []*md.MDTableMeta{{
			DB:         "db",
			Name:       "tbl",
			SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "db", Name: "tbl"}},
			DataFiles:  []md.FileInfo{{TableName: filter.Table{Schema: "db", Name: "tbl"}, FileMeta: md.SourceFileMeta{Path: "db.tbl.sql", Type: md.SourceTypeSQL}}},
		}},
	}})
}

func (s *testMydumpLoaderSuite) TestTablesWithDots(c *C) {
	s.touch(c, "db-schema-create.sql")
	s.touch(c, "db.tbl.with.dots-schema.sql")
	s.touch(c, "db.tbl.with.dots.0001.sql")
	s.touch(c, "db.0002-schema.sql")
	s.touch(c, "db.0002.sql")

	// insert some tables with file name structures which we're going to ignore.
	s.touch(c, "db.v-schema-trigger.sql")
	s.touch(c, "db.v-schema-post.sql")
	s.touch(c, "db.sql")
	s.touch(c, "db-schema.sql")

	mdl, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, IsNil)
	c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{{
		Name:       "db",
		SchemaFile: "db-schema-create.sql",
		Tables: []*md.MDTableMeta{
			{
				DB:         "db",
				Name:       "0002",
				SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "db", Name: "0002"}, FileMeta: md.SourceFileMeta{Path: "db.0002-schema.sql", Type: md.SourceTypeTableSchema}},
				DataFiles:  []md.FileInfo{{TableName: filter.Table{Schema: "db", Name: "0002"}, FileMeta: md.SourceFileMeta{Path: "db.0002.sql", Type: md.SourceTypeSQL}}},
			},
			{
				DB:         "db",
				Name:       "tbl.with.dots",
				SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "db", Name: "tbl.with.dots"}, FileMeta: md.SourceFileMeta{Path: "db.tbl.with.dots-schema.sql", Type: md.SourceTypeTableSchema}},
				DataFiles:  []md.FileInfo{{TableName: filter.Table{Schema: "db", Name: "tbl.with.dots"}, FileMeta: md.SourceFileMeta{Path: "db.tbl.with.dots.0001.sql", Type: md.SourceTypeSQL, SortKey: "0001"}}},
			},
		},
	}})
}

func (s *testMydumpLoaderSuite) TestRouter(c *C) {
	s.cfg.Routes = []*router.TableRule{
		{
			SchemaPattern: "a*",
			TablePattern:  "t*",
			TargetSchema:  "b",
			TargetTable:   "u",
		},
		{
			SchemaPattern: "c*",
			TargetSchema:  "c",
		},
		{
			SchemaPattern: "e*",
			TablePattern:  "f*",
			TargetSchema:  "v",
			TargetTable:   "vv",
		},
	}

	/*
		Path/
			a0-schema-create.sql
			a0.t0-schema.sql
			a0.t0.1.sql
			a0.t1-schema.sql
			a0.t1.1.sql
			a1-schema-create.sql
			a1.s1-schema.sql
			a1.s1.1.schema.sql
			a1.t2-schema.sql
			a1.t2.1.sql
			a1.v1-schema.sql
			a1.v1-schema-view.sql
			c0-schema-create.sql
			c0.t3-schema.sql
			c0.t3.1.sql
			d0-schema-create.sql
			e0-schema-create.sql
			e0.f0-schema.sql
			e0.f0-schema-view.sql
	*/

	s.touch(c, "a0-schema-create.sql")
	s.touch(c, "a0.t0-schema.sql")
	s.touch(c, "a0.t0.1.sql")
	s.touch(c, "a0.t1-schema.sql")
	s.touch(c, "a0.t1.1.sql")

	s.touch(c, "a1-schema-create.sql")
	s.touch(c, "a1.s1-schema.sql")
	s.touch(c, "a1.s1.1.sql")
	s.touch(c, "a1.t2-schema.sql")
	s.touch(c, "a1.t2.1.sql")
	s.touch(c, "a1.v1-schema.sql")
	s.touch(c, "a1.v1-schema-view.sql")

	s.touch(c, "c0-schema-create.sql")
	s.touch(c, "c0.t3-schema.sql")
	s.touch(c, "c0.t3.1.sql")

	s.touch(c, "d0-schema-create.sql")

	s.touch(c, "e0-schema-create.sql")
	s.touch(c, "e0.f0-schema.sql")
	s.touch(c, "e0.f0-schema-view.sql")

	mdl, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, IsNil)
	c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{
		{
			Name:       "a1",
			SchemaFile: "a1-schema-create.sql",
			Tables: []*md.MDTableMeta{
				{
					DB:         "a1",
					Name:       "s1",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: "s1"}, FileMeta: md.SourceFileMeta{Path: "a1.s1-schema.sql", Type: md.SourceTypeTableSchema}},
					DataFiles:  []md.FileInfo{{TableName: filter.Table{Schema: "a1", Name: "s1"}, FileMeta: md.SourceFileMeta{Path: "a1.s1.1.sql", Type: md.SourceTypeSQL, SortKey: "1"}}},
				},
				{
					DB:         "a1",
					Name:       "v1",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: "v1"}, FileMeta: md.SourceFileMeta{Path: "a1.v1-schema.sql", Type: md.SourceTypeTableSchema}},
					DataFiles:  []md.FileInfo{},
				},
			},
			Views: []*md.MDTableMeta{
				{
					DB:         "a1",
					Name:       "v1",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: "v1"}, FileMeta: md.SourceFileMeta{Path: "a1.v1-schema-view.sql", Type: md.SourceTypeViewSchema}},
				},
			},
		},
		{
			Name:       "d0",
			SchemaFile: "d0-schema-create.sql",
		},
		{
			Name:       "b",
			SchemaFile: "a0-schema-create.sql",
			Tables: []*md.MDTableMeta{
				{
					DB:         "b",
					Name:       "u",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "b", Name: "u"}, FileMeta: md.SourceFileMeta{Path: "a0.t0-schema.sql", Type: md.SourceTypeTableSchema}},
					DataFiles: []md.FileInfo{
						{TableName: filter.Table{Schema: "b", Name: "u"}, FileMeta: md.SourceFileMeta{Path: "a0.t0.1.sql", Type: md.SourceTypeSQL, SortKey: "1"}},
						{TableName: filter.Table{Schema: "b", Name: "u"}, FileMeta: md.SourceFileMeta{Path: "a0.t1.1.sql", Type: md.SourceTypeSQL, SortKey: "1"}},
						{TableName: filter.Table{Schema: "b", Name: "u"}, FileMeta: md.SourceFileMeta{Path: "a1.t2.1.sql", Type: md.SourceTypeSQL, SortKey: "1"}},
					},
				},
			},
		},
		{
			Name:       "c",
			SchemaFile: "c0-schema-create.sql",
			Tables: []*md.MDTableMeta{
				{
					DB:         "c",
					Name:       "t3",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "c", Name: "t3"}, FileMeta: md.SourceFileMeta{Path: "c0.t3-schema.sql", Type: md.SourceTypeTableSchema}},
					DataFiles:  []md.FileInfo{{TableName: filter.Table{Schema: "c", Name: "t3"}, FileMeta: md.SourceFileMeta{Path: "c0.t3.1.sql", Type: md.SourceTypeSQL, SortKey: "1"}}},
				},
			},
		},
		{
			Name:       "v",
			SchemaFile: "e0-schema-create.sql",
			Tables: []*md.MDTableMeta{
				{
					DB:         "v",
					Name:       "vv",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "v", Name: "vv"}, FileMeta: md.SourceFileMeta{Path: "e0.f0-schema.sql", Type: md.SourceTypeTableSchema}},
					DataFiles:  []md.FileInfo{},
				},
			},
			Views: []*md.MDTableMeta{
				{
					DB:         "v",
					Name:       "vv",
					SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "v", Name: "vv"}, FileMeta: md.SourceFileMeta{Path: "e0.f0-schema-view.sql", Type: md.SourceTypeViewSchema}},
				},
			},
		},
	})
}

func (s *testMydumpLoaderSuite) TestBadRouterRule(c *C) {
	s.cfg.Routes = []*router.TableRule{{
		SchemaPattern: "a*b",
		TargetSchema:  "ab",
	}}

	s.touch(c, "a1b-schema-create.sql")

	_, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, ErrorMatches, `.*pattern a\*b not valid`)
}

func (s *testMydumpLoaderSuite) TestFileRouting(c *C) {
	s.cfg.Mydumper.DefaultFileRules = false
	s.cfg.Mydumper.FileRouters = []*config.FileRouteRule{
		{
			Pattern: `(?i)^(?:[^./]*/)*([a-z0-9_]+)/schema\.sql$`,
			Schema:  "$1",
			Type:    "schema-schema",
		},
		{
			Pattern: `(?i)^(?:[^./]*/)*([a-z0-9]+)/([a-z0-9_]+)-table\.sql$`,
			Schema:  "$1",
			Table:   "$2",
			Type:    "table-schema",
		},
		{
			Pattern: `(?i)^(?:[^./]*/)*([a-z0-9]+)/([a-z0-9_]+)-view\.sql$`,
			Schema:  "$1",
			Table:   "$2",
			Type:    "view-schema",
		},
		{
			Pattern: `(?i)^(?:[^./]*/)*([a-z][a-z0-9_]*)/([a-z]+)[0-9]*(?:\.([0-9]+))?\.(sql|csv)$`,
			Schema:  "$1",
			Table:   "$2",
			Type:    "$4",
		},
		{
			Pattern: `^(?:[^./]*/)*([a-z]+)(?:\.([0-9]+))?\.(sql|csv)$`,
			Schema:  "d2",
			Table:   "$1",
			Type:    "$3",
		},
	}

	s.mkdir(c, "d1")
	s.mkdir(c, "d2")
	s.touch(c, "d1/schema.sql")
	s.touch(c, "d1/test-table.sql")
	s.touch(c, "d1/test0.sql")
	s.touch(c, "d1/test1.sql")
	s.touch(c, "d1/test2.001.sql")
	s.touch(c, "d1/v1-table.sql")
	s.touch(c, "d1/v1-view.sql")
	_ = s.touch(c, "d1/t1-schema-create.sql")
	s.touch(c, "d2/schema.sql")
	s.touch(c, "d2/abc-table.sql")
	s.touch(c, "abc.1.sql")

	mdl, err := md.NewMyDumpLoader(context.Background(), s.cfg)
	c.Assert(err, IsNil)
	c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{
		{
			Name:       "d1",
			SchemaFile: filepath.FromSlash("d1/schema.sql"),
			Tables: []*md.MDTableMeta{
				{
					DB:   "d1",
					Name: "test",
					SchemaFile: md.FileInfo{
						TableName: filter.Table{Schema: "d1", Name: "test"},
						FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d1/test-table.sql"), Type: md.SourceTypeTableSchema},
					},
					DataFiles: []md.FileInfo{
						{
							TableName: filter.Table{Schema: "d1", Name: "test"},
							FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d1/test0.sql"), Type: md.SourceTypeSQL},
						},
						{
							TableName: filter.Table{Schema: "d1", Name: "test"},
							FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d1/test1.sql"), Type: md.SourceTypeSQL},
						},
						{
							TableName: filter.Table{Schema: "d1", Name: "test"},
							FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d1/test2.001.sql"), Type: md.SourceTypeSQL},
						},
					},
				},
				{
					DB:   "d1",
					Name: "v1",
					SchemaFile: md.FileInfo{
						TableName: filter.Table{Schema: "d1", Name: "v1"},
						FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d1/v1-table.sql"), Type: md.SourceTypeTableSchema},
					},
					DataFiles: []md.FileInfo{},
				},
			},
			Views: []*md.MDTableMeta{
				{
					DB:   "d1",
					Name: "v1",
					SchemaFile: md.FileInfo{
						TableName: filter.Table{Schema: "d1", Name: "v1"},
						FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d1/v1-view.sql"), Type: md.SourceTypeViewSchema},
					},
				},
			},
		},
		{
			Name:       "d2",
			SchemaFile: filepath.FromSlash("d2/schema.sql"),
			Tables: []*md.MDTableMeta{
				{
					DB:   "d2",
					Name: "abc",
					SchemaFile: md.FileInfo{
						TableName: filter.Table{Schema: "d2", Name: "abc"},
						FileMeta:  md.SourceFileMeta{Path: filepath.FromSlash("d2/abc-table.sql"), Type: md.SourceTypeTableSchema},
					},
					DataFiles: []md.FileInfo{{TableName: filter.Table{Schema: "d2", Name: "abc"}, FileMeta: md.SourceFileMeta{Path: "abc.1.sql", Type: md.SourceTypeSQL}}},
				},
			},
		},
	})
}