github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/pkg/backup/schema_test.go (about) 1 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. 2 3 package backup_test 4 5 import ( 6 "context" 7 "fmt" 8 "math" 9 "strings" 10 "sync/atomic" 11 12 "github.com/golang/protobuf/proto" 13 backuppb "github.com/pingcap/kvproto/pkg/backup" 14 15 . "github.com/pingcap/check" 16 filter "github.com/pingcap/tidb-tools/pkg/table-filter" 17 "github.com/pingcap/tidb/sessionctx/variable" 18 "github.com/pingcap/tidb/util/testkit" 19 "github.com/pingcap/tidb/util/testleak" 20 21 "github.com/pingcap/br/pkg/backup" 22 "github.com/pingcap/br/pkg/metautil" 23 "github.com/pingcap/br/pkg/mock" 24 "github.com/pingcap/br/pkg/storage" 25 "github.com/pingcap/br/pkg/utils" 26 ) 27 28 var _ = Suite(&testBackupSchemaSuite{}) 29 30 type testBackupSchemaSuite struct { 31 mock *mock.Cluster 32 } 33 34 func (s *testBackupSchemaSuite) SetUpSuite(c *C) { 35 var err error 36 s.mock, err = mock.NewCluster() 37 c.Assert(err, IsNil) 38 c.Assert(s.mock.Start(), IsNil) 39 } 40 41 func (s *testBackupSchemaSuite) TearDownSuite(c *C) { 42 s.mock.Stop() 43 testleak.AfterTest(c)() 44 } 45 46 func (s *testBackupSchemaSuite) GetRandomStorage(c *C) storage.ExternalStorage { 47 base := c.MkDir() 48 es, err := storage.NewLocalStorage(base) 49 c.Assert(err, IsNil) 50 return es 51 } 52 53 func (s *testBackupSchemaSuite) GetSchemasFromMeta(c *C, es storage.ExternalStorage) []*metautil.Table { 54 ctx := context.Background() 55 metaBytes, err := es.ReadFile(ctx, metautil.MetaFile) 56 c.Assert(err, IsNil) 57 mockMeta := &backuppb.BackupMeta{} 58 err = proto.Unmarshal(metaBytes, mockMeta) 59 c.Assert(err, IsNil) 60 metaReader := metautil.NewMetaReader(mockMeta, es) 61 62 output := make(chan *metautil.Table, 4) 63 go func() { 64 err = metaReader.ReadSchemasFiles(ctx, output) 65 c.Assert(err, IsNil) 66 close(output) 67 }() 68 69 schemas := make([]*metautil.Table, 0, 4) 70 for s := range output { 71 schemas = append(schemas, s) 72 } 73 return schemas 74 } 75 76 type simpleProgress struct { 77 counter int64 78 } 79 80 func (sp *simpleProgress) Inc() { 81 atomic.AddInt64(&sp.counter, 1) 82 } 83 84 func (sp *simpleProgress) Close() {} 85 86 func (sp *simpleProgress) reset() { 87 atomic.StoreInt64(&sp.counter, 0) 88 } 89 90 func (sp *simpleProgress) get() int64 { 91 return atomic.LoadInt64(&sp.counter) 92 } 93 94 func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { 95 tk := testkit.NewTestKit(c, s.mock.Storage) 96 97 // Table t1 is not exist. 98 testFilter, err := filter.Parse([]string{"test.t1"}) 99 c.Assert(err, IsNil) 100 _, backupSchemas, err := backup.BuildBackupRangeAndSchema( 101 s.mock.Storage, testFilter, math.MaxUint64) 102 c.Assert(err, IsNil) 103 c.Assert(backupSchemas, IsNil) 104 105 // Database is not exist. 106 fooFilter, err := filter.Parse([]string{"foo.t1"}) 107 c.Assert(err, IsNil) 108 _, backupSchemas, err = backup.BuildBackupRangeAndSchema( 109 s.mock.Storage, fooFilter, math.MaxUint64) 110 c.Assert(err, IsNil) 111 c.Assert(backupSchemas, IsNil) 112 113 // Empty database. 114 // Filter out system tables manually. 115 noFilter, err := filter.Parse([]string{"*.*", "!mysql.*"}) 116 c.Assert(err, IsNil) 117 _, backupSchemas, err = backup.BuildBackupRangeAndSchema( 118 s.mock.Storage, noFilter, math.MaxUint64) 119 c.Assert(err, IsNil) 120 c.Assert(backupSchemas, IsNil) 121 122 tk.MustExec("use test") 123 tk.MustExec("drop table if exists t1;") 124 tk.MustExec("create table t1 (a int);") 125 tk.MustExec("insert into t1 values (10);") 126 127 _, backupSchemas, err = backup.BuildBackupRangeAndSchema( 128 s.mock.Storage, testFilter, math.MaxUint64) 129 c.Assert(err, IsNil) 130 c.Assert(backupSchemas.Len(), Equals, 1) 131 updateCh := new(simpleProgress) 132 skipChecksum := false 133 es := s.GetRandomStorage(c) 134 metaWriter := metautil.NewMetaWriter(es, metautil.MetaFileSize, false) 135 ctx := context.Background() 136 err = backupSchemas.BackupSchemas( 137 ctx, metaWriter, s.mock.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) 138 c.Assert(updateCh.get(), Equals, int64(1)) 139 c.Assert(err, IsNil) 140 141 schemas := s.GetSchemasFromMeta(c, es) 142 c.Assert(len(schemas), Equals, 1) 143 // Cluster returns a dummy checksum (all fields are 1). 144 c.Assert(schemas[0].Crc64Xor, Not(Equals), 0, Commentf("%v", schemas[0])) 145 c.Assert(schemas[0].TotalKvs, Not(Equals), 0, Commentf("%v", schemas[0])) 146 c.Assert(schemas[0].TotalBytes, Not(Equals), 0, Commentf("%v", schemas[0])) 147 148 tk.MustExec("drop table if exists t2;") 149 tk.MustExec("create table t2 (a int);") 150 tk.MustExec("insert into t2 values (10);") 151 tk.MustExec("insert into t2 values (11);") 152 153 _, backupSchemas, err = backup.BuildBackupRangeAndSchema( 154 s.mock.Storage, noFilter, math.MaxUint64) 155 c.Assert(err, IsNil) 156 c.Assert(backupSchemas.Len(), Equals, 2) 157 updateCh.reset() 158 159 es2 := s.GetRandomStorage(c) 160 metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false) 161 err = backupSchemas.BackupSchemas( 162 ctx, metaWriter2, s.mock.Storage, nil, math.MaxUint64, 2, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) 163 c.Assert(updateCh.get(), Equals, int64(2)) 164 c.Assert(err, IsNil) 165 166 schemas = s.GetSchemasFromMeta(c, es2) 167 168 c.Assert(len(schemas), Equals, 2) 169 // Cluster returns a dummy checksum (all fields are 1). 170 c.Assert(schemas[0].Crc64Xor, Not(Equals), 0, Commentf("%v", schemas[0])) 171 c.Assert(schemas[0].TotalKvs, Not(Equals), 0, Commentf("%v", schemas[0])) 172 c.Assert(schemas[0].TotalBytes, Not(Equals), 0, Commentf("%v", schemas[0])) 173 c.Assert(schemas[1].Crc64Xor, Not(Equals), 0, Commentf("%v", schemas[1])) 174 c.Assert(schemas[1].TotalKvs, Not(Equals), 0, Commentf("%v", schemas[1])) 175 c.Assert(schemas[1].TotalBytes, Not(Equals), 0, Commentf("%v", schemas[1])) 176 } 177 178 func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchemaWithBrokenStats(c *C) { 179 tk := testkit.NewTestKit(c, s.mock.Storage) 180 tk.MustExec("use test") 181 tk.MustExec("drop table if exists t3;") 182 tk.MustExec("create table t3 (a char(1));") 183 tk.MustExec("insert into t3 values ('1');") 184 tk.MustExec("analyze table t3;") 185 // corrupt the statistics like pingcap/br#679. 186 tk.MustExec(` 187 update mysql.stats_buckets set upper_bound = 0xffffffff 188 where table_id = ( 189 select tidb_table_id from information_schema.tables 190 where (table_schema, table_name) = ('test', 't3') 191 ); 192 `) 193 194 f, err := filter.Parse([]string{"test.t3"}) 195 c.Assert(err, IsNil) 196 197 _, backupSchemas, err := backup.BuildBackupRangeAndSchema(s.mock.Storage, f, math.MaxUint64) 198 c.Assert(err, IsNil) 199 c.Assert(backupSchemas.Len(), Equals, 1) 200 201 skipChecksum := false 202 updateCh := new(simpleProgress) 203 204 es := s.GetRandomStorage(c) 205 metaWriter := metautil.NewMetaWriter(es, metautil.MetaFileSize, false) 206 ctx := context.Background() 207 err = backupSchemas.BackupSchemas( 208 ctx, metaWriter, s.mock.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) 209 210 schemas := s.GetSchemasFromMeta(c, es) 211 c.Assert(err, IsNil) 212 c.Assert(schemas, HasLen, 1) 213 // the stats should be empty, but other than that everything should be backed up. 214 c.Assert(schemas[0].Stats, IsNil) 215 c.Assert(schemas[0].Crc64Xor, Not(Equals), 0) 216 c.Assert(schemas[0].TotalKvs, Not(Equals), 0) 217 c.Assert(schemas[0].TotalBytes, Not(Equals), 0) 218 c.Assert(schemas[0].Info, NotNil) 219 c.Assert(schemas[0].DB, NotNil) 220 221 // recover the statistics. 222 tk.MustExec("analyze table t3;") 223 224 _, backupSchemas, err = backup.BuildBackupRangeAndSchema(s.mock.Storage, f, math.MaxUint64) 225 c.Assert(err, IsNil) 226 c.Assert(backupSchemas.Len(), Equals, 1) 227 228 updateCh.reset() 229 statsHandle := s.mock.Domain.StatsHandle() 230 es2 := s.GetRandomStorage(c) 231 metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false) 232 err = backupSchemas.BackupSchemas( 233 ctx, metaWriter2, s.mock.Storage, statsHandle, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) 234 c.Assert(err, IsNil) 235 236 schemas2 := s.GetSchemasFromMeta(c, es2) 237 c.Assert(schemas2, HasLen, 1) 238 // the stats should now be filled, and other than that the result should be equivalent to the first backup. 239 c.Assert(schemas2[0].Stats, NotNil) 240 c.Assert(schemas2[0].Crc64Xor, Equals, schemas[0].Crc64Xor) 241 c.Assert(schemas2[0].TotalKvs, Equals, schemas[0].TotalKvs) 242 c.Assert(schemas2[0].TotalBytes, Equals, schemas[0].TotalBytes) 243 c.Assert(schemas2[0].Info, DeepEquals, schemas[0].Info) 244 c.Assert(schemas2[0].DB, DeepEquals, schemas[0].DB) 245 } 246 247 func (s *testBackupSchemaSuite) TestBackupSchemasForSystemTable(c *C) { 248 tk := testkit.NewTestKit(c, s.mock.Storage) 249 es2 := s.GetRandomStorage(c) 250 251 systemTablesCount := 32 252 tablePrefix := "systable" 253 tk.MustExec("use mysql") 254 for i := 1; i <= systemTablesCount; i++ { 255 query := fmt.Sprintf("create table %s%d (a char(1));", tablePrefix, i) 256 tk.MustExec(query) 257 } 258 259 f, err := filter.Parse([]string{"mysql.systable*"}) 260 c.Assert(err, IsNil) 261 _, backupSchemas, err := backup.BuildBackupRangeAndSchema(s.mock.Storage, f, math.MaxUint64) 262 c.Assert(err, IsNil) 263 c.Assert(backupSchemas.Len(), Equals, systemTablesCount) 264 265 ctx := context.Background() 266 updateCh := new(simpleProgress) 267 268 metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false) 269 err = backupSchemas.BackupSchemas(ctx, metaWriter2, s.mock.Storage, nil, 270 math.MaxUint64, 1, variable.DefChecksumTableConcurrency, true, updateCh) 271 c.Assert(err, IsNil) 272 273 schemas2 := s.GetSchemasFromMeta(c, es2) 274 c.Assert(schemas2, HasLen, systemTablesCount) 275 for _, schema := range schemas2 { 276 c.Assert(schema.DB.Name, Equals, utils.TemporaryDBName("mysql")) 277 c.Assert(strings.HasPrefix(schema.Info.Name.O, tablePrefix), Equals, true) 278 } 279 }