github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/pkg/lightning/backend/kv/sql2kv_test.go (about) 1 // Copyright 2019 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package kv 15 16 import ( 17 "errors" 18 "fmt" 19 20 . "github.com/pingcap/check" 21 "github.com/pingcap/parser" 22 "github.com/pingcap/parser/ast" 23 "github.com/pingcap/parser/model" 24 "github.com/pingcap/parser/mysql" 25 "github.com/pingcap/tidb/ddl" 26 "github.com/pingcap/tidb/kv" 27 "github.com/pingcap/tidb/meta/autoid" 28 "github.com/pingcap/tidb/sessionctx" 29 "github.com/pingcap/tidb/table" 30 "github.com/pingcap/tidb/table/tables" 31 "github.com/pingcap/tidb/tablecodec" 32 "github.com/pingcap/tidb/types" 33 "github.com/pingcap/tidb/util/mock" 34 "go.uber.org/zap" 35 "go.uber.org/zap/zapcore" 36 37 "github.com/pingcap/br/pkg/lightning/common" 38 "github.com/pingcap/br/pkg/lightning/log" 39 "github.com/pingcap/br/pkg/lightning/verification" 40 ) 41 42 func (s *kvSuite) TestMarshal(c *C) { 43 nullDatum := types.Datum{} 44 nullDatum.SetNull() 45 minNotNull := types.Datum{} 46 minNotNull.SetMinNotNull() 47 encoder := zapcore.NewMapObjectEncoder() 48 err := encoder.AddArray("test", RowArrayMarshaler{types.NewStringDatum("1"), nullDatum, minNotNull, types.MaxValueDatum()}) 49 c.Assert(err, IsNil) 50 c.Assert(encoder.Fields["test"], DeepEquals, []interface{}{ 51 map[string]interface{}{"kind": "string", "val": "1"}, 52 map[string]interface{}{"kind": "null", "val": "NULL"}, 53 map[string]interface{}{"kind": "min", "val": "-inf"}, 54 map[string]interface{}{"kind": "max", "val": "+inf"}, 55 }) 56 57 invalid := types.Datum{} 58 invalid.SetInterface(1) 59 err = encoder.AddArray("bad-test", RowArrayMarshaler{minNotNull, invalid}) 60 c.Assert(err, ErrorMatches, "cannot convert.*") 61 c.Assert(encoder.Fields["bad-test"], DeepEquals, []interface{}{ 62 map[string]interface{}{"kind": "min", "val": "-inf"}, 63 }) 64 } 65 66 type mockTable struct { 67 table.Table 68 } 69 70 func (mockTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) { 71 return kv.IntHandle(-1), errors.New("mock error") 72 } 73 74 func (s *kvSuite) TestEncode(c *C) { 75 c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} 76 cols := []*model.ColumnInfo{c1} 77 tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} 78 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 79 c.Assert(err, IsNil) 80 81 logger := log.Logger{Logger: zap.NewNop()} 82 rows := []types.Datum{ 83 types.NewIntDatum(10000000), 84 } 85 86 // Strict mode 87 strictMode, err := NewTableKVEncoder(tbl, &SessionOptions{ 88 SQLMode: mysql.ModeStrictAllTables, 89 Timestamp: 1234567890, 90 }) 91 c.Assert(err, IsNil) 92 pairs, err := strictMode.Encode(logger, rows, 1, []int{0, 1}, 1234) 93 c.Assert(err, ErrorMatches, "failed to cast value as tinyint\\(4\\) for column `c1` \\(#1\\):.*overflows tinyint") 94 c.Assert(pairs, IsNil) 95 96 rowsWithPk := []types.Datum{ 97 types.NewIntDatum(1), 98 types.NewStringDatum("invalid-pk"), 99 } 100 _, err = strictMode.Encode(logger, rowsWithPk, 2, []int{0, 1}, 1234) 101 c.Assert(err, ErrorMatches, "failed to cast value as bigint\\(20\\) for column `_tidb_rowid`.*Truncated.*") 102 103 rowsWithPk2 := []types.Datum{ 104 types.NewIntDatum(1), 105 types.NewStringDatum("1"), 106 } 107 pairs, err = strictMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}, 1234) 108 c.Assert(err, IsNil) 109 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 110 { 111 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, 112 Val: []uint8{0x8, 0x2, 0x8, 0x2}, 113 RowID: 2, 114 Offset: 1234, 115 }, 116 }}) 117 118 // Mock add record error 119 mockTbl := &mockTable{Table: tbl} 120 mockMode, err := NewTableKVEncoder(mockTbl, &SessionOptions{ 121 SQLMode: mysql.ModeStrictAllTables, 122 Timestamp: 1234567891, 123 }) 124 c.Assert(err, IsNil) 125 _, err = mockMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}, 1234) 126 c.Assert(err, ErrorMatches, "mock error") 127 128 // Non-strict mode 129 noneMode, err := NewTableKVEncoder(tbl, &SessionOptions{ 130 SQLMode: mysql.ModeNone, 131 Timestamp: 1234567892, 132 SysVars: map[string]string{"tidb_row_format_version": "1"}, 133 }) 134 c.Assert(err, IsNil) 135 pairs, err = noneMode.Encode(logger, rows, 1, []int{0, 1}, 1234) 136 c.Assert(err, IsNil) 137 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 138 { 139 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, 140 Val: []uint8{0x8, 0x2, 0x8, 0xfe, 0x1}, 141 RowID: 1, 142 Offset: 1234, 143 }, 144 }}) 145 } 146 147 func (s *kvSuite) TestDecode(c *C) { 148 c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} 149 cols := []*model.ColumnInfo{c1} 150 tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} 151 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 152 c.Assert(err, IsNil) 153 decoder, err := NewTableKVDecoder(tbl, &SessionOptions{ 154 SQLMode: mysql.ModeStrictAllTables, 155 Timestamp: 1234567890, 156 }) 157 c.Assert(decoder, NotNil) 158 p := common.KvPair{ 159 Key: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, 160 Val: []byte{0x8, 0x2, 0x8, 0x2}, 161 } 162 h, err := decoder.DecodeHandleFromTable(p.Key) 163 c.Assert(err, IsNil) 164 c.Assert(p.Val, NotNil) 165 rows, _, err := decoder.DecodeRawRowData(h, p.Val) 166 c.Assert(rows, DeepEquals, []types.Datum{ 167 types.NewIntDatum(1), 168 }) 169 } 170 171 func (s *kvSuite) TestDecodeIndex(c *C) { 172 logger := log.Logger{Logger: zap.NewNop()} 173 tblInfo := &model.TableInfo{ 174 ID: 1, 175 Indices: []*model.IndexInfo{ 176 { 177 ID: 2, 178 Name: model.NewCIStr("test"), 179 Columns: []*model.IndexColumn{ 180 {Offset: 0}, 181 {Offset: 1}, 182 }, 183 Primary: true, 184 State: model.StatePublic, 185 }, 186 }, 187 Columns: []*model.ColumnInfo{ 188 {ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeInt24)}, 189 {ID: 2, Name: model.NewCIStr("c2"), State: model.StatePublic, Offset: 1, FieldType: *types.NewFieldType(mysql.TypeString)}, 190 }, 191 State: model.StatePublic, 192 PKIsHandle: false, 193 } 194 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 195 if err != nil { 196 fmt.Printf("error: %v", err.Error()) 197 } 198 c.Assert(err, IsNil) 199 rows := []types.Datum{ 200 types.NewIntDatum(2), 201 types.NewStringDatum("abc"), 202 } 203 204 // Strict mode 205 strictMode, err := NewTableKVEncoder(tbl, &SessionOptions{ 206 SQLMode: mysql.ModeStrictAllTables, 207 Timestamp: 1234567890, 208 }) 209 c.Assert(err, IsNil) 210 pairs, err := strictMode.Encode(logger, rows, 1, []int{0, 1, -1}, 123) 211 data := pairs.(*KvPairs) 212 c.Assert(len(data.pairs), DeepEquals, 2) 213 214 decoder, err := NewTableKVDecoder(tbl, &SessionOptions{ 215 SQLMode: mysql.ModeStrictAllTables, 216 Timestamp: 1234567890, 217 }) 218 c.Assert(err, IsNil) 219 h1, err := decoder.DecodeHandleFromTable(data.pairs[0].Key) 220 c.Assert(err, IsNil) 221 h2, err := decoder.DecodeHandleFromIndex(tbl.Indices()[0].Meta(), data.pairs[1].Key, data.pairs[1].Val) 222 c.Assert(err, IsNil) 223 c.Assert(h1.Equal(h2), IsTrue) 224 rawData, _, err := decoder.DecodeRawRowData(h1, data.pairs[0].Val) 225 c.Assert(err, IsNil) 226 c.Assert(rawData, DeepEquals, rows) 227 } 228 229 func (s *kvSuite) TestEncodeRowFormatV2(c *C) { 230 // Test encoding in row format v2, as described in <https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md>. 231 232 c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} 233 cols := []*model.ColumnInfo{c1} 234 tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} 235 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 236 c.Assert(err, IsNil) 237 238 logger := log.Logger{Logger: zap.NewNop()} 239 rows := []types.Datum{ 240 types.NewIntDatum(10000000), 241 } 242 243 noneMode, err := NewTableKVEncoder(tbl, &SessionOptions{ 244 SQLMode: mysql.ModeNone, 245 Timestamp: 1234567892, 246 SysVars: map[string]string{"tidb_row_format_version": "2"}, 247 }) 248 c.Assert(err, IsNil) 249 pairs, err := noneMode.Encode(logger, rows, 1, []int{0, 1}, 1234) 250 c.Assert(err, IsNil) 251 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 252 { 253 // the key should be the same as TestEncode() 254 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, 255 Val: []uint8{ 256 0x80, // version 257 0x0, // flag = 0 = not big 258 0x1, 0x0, // number of not null columns = 1 259 0x0, 0x0, // number of null columns = 0 260 0x1, // column IDs = [1] 261 0x1, 0x0, // not null offsets = [1] 262 0x7f, // column version = 127 (10000000 clamped to TINYINT) 263 }, 264 RowID: 1, 265 Offset: 1234, 266 }, 267 }}) 268 } 269 270 func (s *kvSuite) TestEncodeTimestamp(c *C) { 271 ty := *types.NewFieldType(mysql.TypeDatetime) 272 ty.Flag |= mysql.NotNullFlag 273 c1 := &model.ColumnInfo{ 274 ID: 1, 275 Name: model.NewCIStr("c1"), 276 State: model.StatePublic, 277 Offset: 0, 278 FieldType: ty, 279 DefaultValue: "CURRENT_TIMESTAMP", 280 Version: 1, 281 } 282 cols := []*model.ColumnInfo{c1} 283 tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} 284 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 285 c.Assert(err, IsNil) 286 287 logger := log.Logger{Logger: zap.NewNop()} 288 289 encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ 290 SQLMode: mysql.ModeStrictAllTables, 291 Timestamp: 1234567893, 292 SysVars: map[string]string{ 293 "tidb_row_format_version": "1", 294 "time_zone": "+08:00", 295 }, 296 }) 297 c.Assert(err, IsNil) 298 pairs, err := encoder.Encode(logger, nil, 70, []int{-1, 1}, 1234) 299 c.Assert(err, IsNil) 300 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 301 { 302 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, 303 Val: []uint8{0x8, 0x2, 0x9, 0x80, 0x80, 0x80, 0xf0, 0xfd, 0x8e, 0xf7, 0xc0, 0x19}, 304 RowID: 70, 305 Offset: 1234, 306 }, 307 }}) 308 } 309 310 func (s *kvSuite) TestEncodeDoubleAutoIncrement(c *C) { 311 tblInfo := mockTableInfo(c, "create table t (id double not null auto_increment, unique key `u_id` (`id`));") 312 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 313 c.Assert(err, IsNil) 314 315 logger := log.Logger{Logger: zap.NewNop()} 316 317 encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ 318 SQLMode: mysql.ModeStrictAllTables, 319 SysVars: map[string]string{ 320 "tidb_row_format_version": "2", 321 }, 322 }) 323 c.Assert(err, IsNil) 324 pairs, err := encoder.Encode(logger, []types.Datum{ 325 types.NewStringDatum("1"), 326 }, 70, []int{0, -1}, 1234) 327 c.Assert(err, IsNil) 328 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 329 { 330 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, 331 Val: []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x8, 0x0, 0xbf, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 332 RowID: 70, 333 Offset: 1234, 334 }, 335 { 336 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5, 0xbf, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, 337 Val: []uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, 338 RowID: 70, 339 Offset: 1234, 340 }, 341 }}) 342 c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoIncrementType).Base(), Equals, int64(70)) 343 } 344 345 func mockTableInfo(c *C, createSQL string) *model.TableInfo { 346 parser := parser.New() 347 node, err := parser.ParseOneStmt(createSQL, "", "") 348 c.Assert(err, IsNil) 349 sctx := mock.NewContext() 350 info, err := ddl.MockTableInfo(sctx, node.(*ast.CreateTableStmt), 1) 351 c.Assert(err, IsNil) 352 info.State = model.StatePublic 353 return info 354 } 355 356 func (s *kvSuite) TestDefaultAutoRandoms(c *C) { 357 tblInfo := mockTableInfo(c, "create table t (id bigint unsigned NOT NULL auto_random primary key clustered, a varchar(100));") 358 // seems parser can't parse auto_random properly. 359 tblInfo.AutoRandomBits = 5 360 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 361 c.Assert(err, IsNil) 362 encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ 363 SQLMode: mysql.ModeStrictAllTables, 364 Timestamp: 1234567893, 365 SysVars: map[string]string{"tidb_row_format_version": "2"}, 366 AutoRandomSeed: 456, 367 }) 368 c.Assert(err, IsNil) 369 logger := log.Logger{Logger: zap.NewNop()} 370 pairs, err := encoder.Encode(logger, []types.Datum{types.NewStringDatum("")}, 70, []int{-1, 0}, 1234) 371 c.Assert(err, IsNil) 372 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 373 { 374 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, 375 Val: []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0}, 376 RowID: 70, 377 Offset: 1234, 378 }, 379 }}) 380 c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoRandomType).Base(), Equals, int64(70)) 381 382 pairs, err = encoder.Encode(logger, []types.Datum{types.NewStringDatum("")}, 71, []int{-1, 0}, 1234) 383 c.Assert(err, IsNil) 384 c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{ 385 { 386 Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x47}, 387 Val: []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0}, 388 RowID: 71, 389 Offset: 1234, 390 }, 391 }}) 392 c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoRandomType).Base(), Equals, int64(71)) 393 } 394 395 func (s *kvSuite) TestShardRowId(c *C) { 396 tblInfo := mockTableInfo(c, "create table t (s varchar(16)) shard_row_id_bits = 3;") 397 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) 398 c.Assert(err, IsNil) 399 encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ 400 SQLMode: mysql.ModeStrictAllTables, 401 Timestamp: 1234567893, 402 SysVars: map[string]string{"tidb_row_format_version": "2"}, 403 AutoRandomSeed: 456, 404 }) 405 c.Assert(err, IsNil) 406 logger := log.Logger{Logger: zap.NewNop()} 407 keyMap := make(map[int64]struct{}, 16) 408 for i := int64(1); i <= 32; i++ { 409 pairs, err := encoder.Encode(logger, []types.Datum{types.NewStringDatum(fmt.Sprintf("%d", i))}, i, []int{0, -1}, i*32) 410 c.Assert(err, IsNil) 411 kvs := pairs.(*KvPairs) 412 c.Assert(len(kvs.pairs), Equals, 1) 413 _, h, err := tablecodec.DecodeRecordKey(kvs.pairs[0].Key) 414 c.Assert(err, IsNil) 415 rowID := h.IntValue() 416 c.Assert(rowID&((1<<60)-1), Equals, i) 417 keyMap[rowID>>60] = struct{}{} 418 } 419 c.Assert(len(keyMap), Equals, 8) 420 c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.RowIDAllocType).Base(), Equals, int64(32)) 421 } 422 423 func (s *kvSuite) TestSplitIntoChunks(c *C) { 424 pairs := []common.KvPair{ 425 { 426 Key: []byte{1, 2, 3}, 427 Val: []byte{4, 5, 6}, 428 Offset: 1000, 429 }, 430 { 431 Key: []byte{7, 8}, 432 Val: []byte{9, 0}, 433 Offset: 2000, 434 }, 435 { 436 Key: []byte{1, 2, 3, 4}, 437 Val: []byte{5, 6, 7, 8}, 438 Offset: 3000, 439 }, 440 { 441 Key: []byte{9, 0}, 442 Val: []byte{1, 2}, 443 Offset: 4000, 444 }, 445 } 446 447 splitBy10 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(10) 448 c.Assert(splitBy10, DeepEquals, []Rows{ 449 MakeRowsFromKvPairs(pairs[0:2]), 450 MakeRowsFromKvPairs(pairs[2:3]), 451 MakeRowsFromKvPairs(pairs[3:4]), 452 }) 453 454 splitBy12 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(12) 455 c.Assert(splitBy12, DeepEquals, []Rows{ 456 MakeRowsFromKvPairs(pairs[0:2]), 457 MakeRowsFromKvPairs(pairs[2:4]), 458 }) 459 460 splitBy1000 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(1000) 461 c.Assert(splitBy1000, DeepEquals, []Rows{ 462 MakeRowsFromKvPairs(pairs[0:4]), 463 }) 464 465 splitBy1 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(1) 466 c.Assert(splitBy1, DeepEquals, []Rows{ 467 MakeRowsFromKvPairs(pairs[0:1]), 468 MakeRowsFromKvPairs(pairs[1:2]), 469 MakeRowsFromKvPairs(pairs[2:3]), 470 MakeRowsFromKvPairs(pairs[3:4]), 471 }) 472 } 473 474 func (s *kvSuite) TestClassifyAndAppend(c *C) { 475 kvs := MakeRowFromKvPairs([]common.KvPair{ 476 { 477 Key: []byte("txxxxxxxx_ryyyyyyyy"), 478 Val: []byte("value1"), 479 }, 480 { 481 Key: []byte("txxxxxxxx_rwwwwwwww"), 482 Val: []byte("value2"), 483 }, 484 { 485 Key: []byte("txxxxxxxx_izzzzzzzz"), 486 Val: []byte("index1"), 487 }, 488 }) 489 490 data := MakeRowsFromKvPairs(nil) 491 indices := MakeRowsFromKvPairs(nil) 492 dataChecksum := verification.MakeKVChecksum(0, 0, 0) 493 indexChecksum := verification.MakeKVChecksum(0, 0, 0) 494 495 kvs.ClassifyAndAppend(&data, &dataChecksum, &indices, &indexChecksum) 496 497 c.Assert(data, DeepEquals, MakeRowsFromKvPairs([]common.KvPair{ 498 { 499 Key: []byte("txxxxxxxx_ryyyyyyyy"), 500 Val: []byte("value1"), 501 }, 502 { 503 Key: []byte("txxxxxxxx_rwwwwwwww"), 504 Val: []byte("value2"), 505 }, 506 })) 507 c.Assert(indices, DeepEquals, MakeRowsFromKvPairs([]common.KvPair{ 508 { 509 Key: []byte("txxxxxxxx_izzzzzzzz"), 510 Val: []byte("index1"), 511 }, 512 })) 513 c.Assert(dataChecksum.SumKVS(), Equals, uint64(2)) 514 c.Assert(indexChecksum.SumKVS(), Equals, uint64(1)) 515 } 516 517 type benchSQL2KVSuite struct { 518 row []types.Datum 519 colPerm []int 520 encoder Encoder 521 logger log.Logger 522 } 523 524 var _ = Suite(&benchSQL2KVSuite{}) 525 526 func (s *benchSQL2KVSuite) SetUpTest(c *C) { 527 // First, create the table info corresponding to TPC-C's "CUSTOMER" table. 528 p := parser.New() 529 se := mock.NewContext() 530 node, err := p.ParseOneStmt(` 531 create table bmsql_customer( 532 c_w_id integer not null, 533 c_d_id integer not null, 534 c_id integer not null, 535 c_discount decimal(4,4), 536 c_credit char(2), 537 c_last varchar(16), 538 c_first varchar(16), 539 c_credit_lim decimal(12,2), 540 c_balance decimal(12,2), 541 c_ytd_payment decimal(12,2), 542 c_payment_cnt integer, 543 c_delivery_cnt integer, 544 c_street_1 varchar(20), 545 c_street_2 varchar(20), 546 c_city varchar(20), 547 c_state char(2), 548 c_zip char(9), 549 c_phone char(16), 550 c_since timestamp, 551 c_middle char(2), 552 c_data varchar(500), 553 primary key (c_w_id, c_d_id, c_id) 554 ); 555 `, "", "") 556 c.Assert(err, IsNil) 557 tableInfo, err := ddl.MockTableInfo(se, node.(*ast.CreateTableStmt), 123456) 558 c.Assert(err, IsNil) 559 tableInfo.State = model.StatePublic 560 561 // Construct the corresponding KV encoder. 562 tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tableInfo) 563 c.Assert(err, IsNil) 564 s.encoder, err = NewTableKVEncoder(tbl, &SessionOptions{SysVars: map[string]string{"tidb_row_format_version": "2"}}) 565 c.Assert(err, IsNil) 566 s.logger = log.Logger{Logger: zap.NewNop()} 567 568 // Prepare the row to insert. 569 s.row = []types.Datum{ 570 types.NewIntDatum(15), 571 types.NewIntDatum(10), 572 types.NewIntDatum(3000), 573 types.NewStringDatum("0.3646"), 574 types.NewStringDatum("GC"), 575 types.NewStringDatum("CALLYPRIANTI"), 576 types.NewStringDatum("Rg6mDFlVnP5yh"), 577 types.NewStringDatum("50000.0"), 578 types.NewStringDatum("-10.0"), 579 types.NewStringDatum("10.0"), 580 types.NewIntDatum(1), 581 types.NewIntDatum(0), 582 types.NewStringDatum("aJK7CuRnE0NUxNHSX"), 583 types.NewStringDatum("Q1rps77cXYoj"), 584 types.NewStringDatum("MigXbS6UoUS"), 585 types.NewStringDatum("UJ"), 586 types.NewStringDatum("638611111"), 587 types.NewStringDatum("7743262784364376"), 588 types.NewStringDatum("2020-02-05 19:29:58.903970"), 589 types.NewStringDatum("OE"), 590 types.NewStringDatum("H5p3dpjp7uu8n1l3j0o1buecfV6FngNNgftpNALDhOzJaSzMCMlrQwXuvLAFPIFg215D3wAYB62kiixIuasfbD729oq8TwgKzPPsx8kHE1b4AdhHwpCml3ELKiwuNGQl7CcBQOiq6aFEMMHzjGwQyXwGey0wutjp2KP3Nd4qj3FHtmHbsD8cJ0pH9TswNmdQBgXsFPZeJJhsG3rTimQpS9Tmn3vNeI9fFas3ClDZuQtBjqoTJlyzmBIYT8HeV3TuS93TNFDaXZpQqh8HsvlPq4uTTLOO9CguiY29zlSmIjkZYtva3iscG3YDOQVLeGpP9dtqEJwlRvJ4oe9jWkvRMlCeslSNEuzLxjUBtJBnGRFAzJF6RMlIWCkdCpIhcnIy3jUEsxTuiAU3hsZxUjLg2dnOG62h5qR"), 591 } 592 s.colPerm = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, -1} 593 } 594 595 // Run `go test github.com/pingcap/br/pkg/lightning/backend -check.b -test.v` to get benchmark result. 596 func (s *benchSQL2KVSuite) BenchmarkSQL2KV(c *C) { 597 for i := 0; i < c.N; i++ { 598 rows, err := s.encoder.Encode(s.logger, s.row, 1, s.colPerm, 0) 599 c.Assert(err, IsNil) 600 c.Assert(rows, HasLen, 2) 601 } 602 }