github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/pkg/lightning/backend/kv/sql2kv_test.go (about)

     1  // Copyright 2019 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package kv
    15  
    16  import (
    17  	"errors"
    18  	"fmt"
    19  
    20  	. "github.com/pingcap/check"
    21  	"github.com/pingcap/parser"
    22  	"github.com/pingcap/parser/ast"
    23  	"github.com/pingcap/parser/model"
    24  	"github.com/pingcap/parser/mysql"
    25  	"github.com/pingcap/tidb/ddl"
    26  	"github.com/pingcap/tidb/kv"
    27  	"github.com/pingcap/tidb/meta/autoid"
    28  	"github.com/pingcap/tidb/sessionctx"
    29  	"github.com/pingcap/tidb/table"
    30  	"github.com/pingcap/tidb/table/tables"
    31  	"github.com/pingcap/tidb/tablecodec"
    32  	"github.com/pingcap/tidb/types"
    33  	"github.com/pingcap/tidb/util/mock"
    34  	"go.uber.org/zap"
    35  	"go.uber.org/zap/zapcore"
    36  
    37  	"github.com/pingcap/br/pkg/lightning/common"
    38  	"github.com/pingcap/br/pkg/lightning/log"
    39  	"github.com/pingcap/br/pkg/lightning/verification"
    40  )
    41  
    42  func (s *kvSuite) TestMarshal(c *C) {
    43  	nullDatum := types.Datum{}
    44  	nullDatum.SetNull()
    45  	minNotNull := types.Datum{}
    46  	minNotNull.SetMinNotNull()
    47  	encoder := zapcore.NewMapObjectEncoder()
    48  	err := encoder.AddArray("test", RowArrayMarshaler{types.NewStringDatum("1"), nullDatum, minNotNull, types.MaxValueDatum()})
    49  	c.Assert(err, IsNil)
    50  	c.Assert(encoder.Fields["test"], DeepEquals, []interface{}{
    51  		map[string]interface{}{"kind": "string", "val": "1"},
    52  		map[string]interface{}{"kind": "null", "val": "NULL"},
    53  		map[string]interface{}{"kind": "min", "val": "-inf"},
    54  		map[string]interface{}{"kind": "max", "val": "+inf"},
    55  	})
    56  
    57  	invalid := types.Datum{}
    58  	invalid.SetInterface(1)
    59  	err = encoder.AddArray("bad-test", RowArrayMarshaler{minNotNull, invalid})
    60  	c.Assert(err, ErrorMatches, "cannot convert.*")
    61  	c.Assert(encoder.Fields["bad-test"], DeepEquals, []interface{}{
    62  		map[string]interface{}{"kind": "min", "val": "-inf"},
    63  	})
    64  }
    65  
    66  type mockTable struct {
    67  	table.Table
    68  }
    69  
    70  func (mockTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) {
    71  	return kv.IntHandle(-1), errors.New("mock error")
    72  }
    73  
    74  func (s *kvSuite) TestEncode(c *C) {
    75  	c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)}
    76  	cols := []*model.ColumnInfo{c1}
    77  	tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic}
    78  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
    79  	c.Assert(err, IsNil)
    80  
    81  	logger := log.Logger{Logger: zap.NewNop()}
    82  	rows := []types.Datum{
    83  		types.NewIntDatum(10000000),
    84  	}
    85  
    86  	// Strict mode
    87  	strictMode, err := NewTableKVEncoder(tbl, &SessionOptions{
    88  		SQLMode:   mysql.ModeStrictAllTables,
    89  		Timestamp: 1234567890,
    90  	})
    91  	c.Assert(err, IsNil)
    92  	pairs, err := strictMode.Encode(logger, rows, 1, []int{0, 1}, 1234)
    93  	c.Assert(err, ErrorMatches, "failed to cast value as tinyint\\(4\\) for column `c1` \\(#1\\):.*overflows tinyint")
    94  	c.Assert(pairs, IsNil)
    95  
    96  	rowsWithPk := []types.Datum{
    97  		types.NewIntDatum(1),
    98  		types.NewStringDatum("invalid-pk"),
    99  	}
   100  	_, err = strictMode.Encode(logger, rowsWithPk, 2, []int{0, 1}, 1234)
   101  	c.Assert(err, ErrorMatches, "failed to cast value as bigint\\(20\\) for column `_tidb_rowid`.*Truncated.*")
   102  
   103  	rowsWithPk2 := []types.Datum{
   104  		types.NewIntDatum(1),
   105  		types.NewStringDatum("1"),
   106  	}
   107  	pairs, err = strictMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}, 1234)
   108  	c.Assert(err, IsNil)
   109  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   110  		{
   111  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
   112  			Val:    []uint8{0x8, 0x2, 0x8, 0x2},
   113  			RowID:  2,
   114  			Offset: 1234,
   115  		},
   116  	}})
   117  
   118  	// Mock add record error
   119  	mockTbl := &mockTable{Table: tbl}
   120  	mockMode, err := NewTableKVEncoder(mockTbl, &SessionOptions{
   121  		SQLMode:   mysql.ModeStrictAllTables,
   122  		Timestamp: 1234567891,
   123  	})
   124  	c.Assert(err, IsNil)
   125  	_, err = mockMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}, 1234)
   126  	c.Assert(err, ErrorMatches, "mock error")
   127  
   128  	// Non-strict mode
   129  	noneMode, err := NewTableKVEncoder(tbl, &SessionOptions{
   130  		SQLMode:   mysql.ModeNone,
   131  		Timestamp: 1234567892,
   132  		SysVars:   map[string]string{"tidb_row_format_version": "1"},
   133  	})
   134  	c.Assert(err, IsNil)
   135  	pairs, err = noneMode.Encode(logger, rows, 1, []int{0, 1}, 1234)
   136  	c.Assert(err, IsNil)
   137  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   138  		{
   139  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
   140  			Val:    []uint8{0x8, 0x2, 0x8, 0xfe, 0x1},
   141  			RowID:  1,
   142  			Offset: 1234,
   143  		},
   144  	}})
   145  }
   146  
   147  func (s *kvSuite) TestDecode(c *C) {
   148  	c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)}
   149  	cols := []*model.ColumnInfo{c1}
   150  	tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic}
   151  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   152  	c.Assert(err, IsNil)
   153  	decoder, err := NewTableKVDecoder(tbl, &SessionOptions{
   154  		SQLMode:   mysql.ModeStrictAllTables,
   155  		Timestamp: 1234567890,
   156  	})
   157  	c.Assert(decoder, NotNil)
   158  	p := common.KvPair{
   159  		Key: []byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
   160  		Val: []byte{0x8, 0x2, 0x8, 0x2},
   161  	}
   162  	h, err := decoder.DecodeHandleFromTable(p.Key)
   163  	c.Assert(err, IsNil)
   164  	c.Assert(p.Val, NotNil)
   165  	rows, _, err := decoder.DecodeRawRowData(h, p.Val)
   166  	c.Assert(rows, DeepEquals, []types.Datum{
   167  		types.NewIntDatum(1),
   168  	})
   169  }
   170  
   171  func (s *kvSuite) TestDecodeIndex(c *C) {
   172  	logger := log.Logger{Logger: zap.NewNop()}
   173  	tblInfo := &model.TableInfo{
   174  		ID: 1,
   175  		Indices: []*model.IndexInfo{
   176  			{
   177  				ID:   2,
   178  				Name: model.NewCIStr("test"),
   179  				Columns: []*model.IndexColumn{
   180  					{Offset: 0},
   181  					{Offset: 1},
   182  				},
   183  				Primary: true,
   184  				State:   model.StatePublic,
   185  			},
   186  		},
   187  		Columns: []*model.ColumnInfo{
   188  			{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeInt24)},
   189  			{ID: 2, Name: model.NewCIStr("c2"), State: model.StatePublic, Offset: 1, FieldType: *types.NewFieldType(mysql.TypeString)},
   190  		},
   191  		State:      model.StatePublic,
   192  		PKIsHandle: false,
   193  	}
   194  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   195  	if err != nil {
   196  		fmt.Printf("error: %v", err.Error())
   197  	}
   198  	c.Assert(err, IsNil)
   199  	rows := []types.Datum{
   200  		types.NewIntDatum(2),
   201  		types.NewStringDatum("abc"),
   202  	}
   203  
   204  	// Strict mode
   205  	strictMode, err := NewTableKVEncoder(tbl, &SessionOptions{
   206  		SQLMode:   mysql.ModeStrictAllTables,
   207  		Timestamp: 1234567890,
   208  	})
   209  	c.Assert(err, IsNil)
   210  	pairs, err := strictMode.Encode(logger, rows, 1, []int{0, 1, -1}, 123)
   211  	data := pairs.(*KvPairs)
   212  	c.Assert(len(data.pairs), DeepEquals, 2)
   213  
   214  	decoder, err := NewTableKVDecoder(tbl, &SessionOptions{
   215  		SQLMode:   mysql.ModeStrictAllTables,
   216  		Timestamp: 1234567890,
   217  	})
   218  	c.Assert(err, IsNil)
   219  	h1, err := decoder.DecodeHandleFromTable(data.pairs[0].Key)
   220  	c.Assert(err, IsNil)
   221  	h2, err := decoder.DecodeHandleFromIndex(tbl.Indices()[0].Meta(), data.pairs[1].Key, data.pairs[1].Val)
   222  	c.Assert(err, IsNil)
   223  	c.Assert(h1.Equal(h2), IsTrue)
   224  	rawData, _, err := decoder.DecodeRawRowData(h1, data.pairs[0].Val)
   225  	c.Assert(err, IsNil)
   226  	c.Assert(rawData, DeepEquals, rows)
   227  }
   228  
   229  func (s *kvSuite) TestEncodeRowFormatV2(c *C) {
   230  	// Test encoding in row format v2, as described in <https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md>.
   231  
   232  	c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)}
   233  	cols := []*model.ColumnInfo{c1}
   234  	tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic}
   235  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   236  	c.Assert(err, IsNil)
   237  
   238  	logger := log.Logger{Logger: zap.NewNop()}
   239  	rows := []types.Datum{
   240  		types.NewIntDatum(10000000),
   241  	}
   242  
   243  	noneMode, err := NewTableKVEncoder(tbl, &SessionOptions{
   244  		SQLMode:   mysql.ModeNone,
   245  		Timestamp: 1234567892,
   246  		SysVars:   map[string]string{"tidb_row_format_version": "2"},
   247  	})
   248  	c.Assert(err, IsNil)
   249  	pairs, err := noneMode.Encode(logger, rows, 1, []int{0, 1}, 1234)
   250  	c.Assert(err, IsNil)
   251  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   252  		{
   253  			// the key should be the same as TestEncode()
   254  			Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
   255  			Val: []uint8{
   256  				0x80,     // version
   257  				0x0,      // flag = 0 = not big
   258  				0x1, 0x0, // number of not null columns = 1
   259  				0x0, 0x0, // number of null columns = 0
   260  				0x1,      // column IDs = [1]
   261  				0x1, 0x0, // not null offsets = [1]
   262  				0x7f, // column version = 127 (10000000 clamped to TINYINT)
   263  			},
   264  			RowID:  1,
   265  			Offset: 1234,
   266  		},
   267  	}})
   268  }
   269  
   270  func (s *kvSuite) TestEncodeTimestamp(c *C) {
   271  	ty := *types.NewFieldType(mysql.TypeDatetime)
   272  	ty.Flag |= mysql.NotNullFlag
   273  	c1 := &model.ColumnInfo{
   274  		ID:           1,
   275  		Name:         model.NewCIStr("c1"),
   276  		State:        model.StatePublic,
   277  		Offset:       0,
   278  		FieldType:    ty,
   279  		DefaultValue: "CURRENT_TIMESTAMP",
   280  		Version:      1,
   281  	}
   282  	cols := []*model.ColumnInfo{c1}
   283  	tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic}
   284  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   285  	c.Assert(err, IsNil)
   286  
   287  	logger := log.Logger{Logger: zap.NewNop()}
   288  
   289  	encoder, err := NewTableKVEncoder(tbl, &SessionOptions{
   290  		SQLMode:   mysql.ModeStrictAllTables,
   291  		Timestamp: 1234567893,
   292  		SysVars: map[string]string{
   293  			"tidb_row_format_version": "1",
   294  			"time_zone":               "+08:00",
   295  		},
   296  	})
   297  	c.Assert(err, IsNil)
   298  	pairs, err := encoder.Encode(logger, nil, 70, []int{-1, 1}, 1234)
   299  	c.Assert(err, IsNil)
   300  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   301  		{
   302  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46},
   303  			Val:    []uint8{0x8, 0x2, 0x9, 0x80, 0x80, 0x80, 0xf0, 0xfd, 0x8e, 0xf7, 0xc0, 0x19},
   304  			RowID:  70,
   305  			Offset: 1234,
   306  		},
   307  	}})
   308  }
   309  
   310  func (s *kvSuite) TestEncodeDoubleAutoIncrement(c *C) {
   311  	tblInfo := mockTableInfo(c, "create table t (id double not null auto_increment, unique key `u_id` (`id`));")
   312  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   313  	c.Assert(err, IsNil)
   314  
   315  	logger := log.Logger{Logger: zap.NewNop()}
   316  
   317  	encoder, err := NewTableKVEncoder(tbl, &SessionOptions{
   318  		SQLMode: mysql.ModeStrictAllTables,
   319  		SysVars: map[string]string{
   320  			"tidb_row_format_version": "2",
   321  		},
   322  	})
   323  	c.Assert(err, IsNil)
   324  	pairs, err := encoder.Encode(logger, []types.Datum{
   325  		types.NewStringDatum("1"),
   326  	}, 70, []int{0, -1}, 1234)
   327  	c.Assert(err, IsNil)
   328  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   329  		{
   330  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46},
   331  			Val:    []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x8, 0x0, 0xbf, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
   332  			RowID:  70,
   333  			Offset: 1234,
   334  		},
   335  		{
   336  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5, 0xbf, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
   337  			Val:    []uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46},
   338  			RowID:  70,
   339  			Offset: 1234,
   340  		},
   341  	}})
   342  	c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoIncrementType).Base(), Equals, int64(70))
   343  }
   344  
   345  func mockTableInfo(c *C, createSQL string) *model.TableInfo {
   346  	parser := parser.New()
   347  	node, err := parser.ParseOneStmt(createSQL, "", "")
   348  	c.Assert(err, IsNil)
   349  	sctx := mock.NewContext()
   350  	info, err := ddl.MockTableInfo(sctx, node.(*ast.CreateTableStmt), 1)
   351  	c.Assert(err, IsNil)
   352  	info.State = model.StatePublic
   353  	return info
   354  }
   355  
   356  func (s *kvSuite) TestDefaultAutoRandoms(c *C) {
   357  	tblInfo := mockTableInfo(c, "create table t (id bigint unsigned NOT NULL auto_random primary key clustered, a varchar(100));")
   358  	// seems parser can't parse auto_random properly.
   359  	tblInfo.AutoRandomBits = 5
   360  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   361  	c.Assert(err, IsNil)
   362  	encoder, err := NewTableKVEncoder(tbl, &SessionOptions{
   363  		SQLMode:        mysql.ModeStrictAllTables,
   364  		Timestamp:      1234567893,
   365  		SysVars:        map[string]string{"tidb_row_format_version": "2"},
   366  		AutoRandomSeed: 456,
   367  	})
   368  	c.Assert(err, IsNil)
   369  	logger := log.Logger{Logger: zap.NewNop()}
   370  	pairs, err := encoder.Encode(logger, []types.Datum{types.NewStringDatum("")}, 70, []int{-1, 0}, 1234)
   371  	c.Assert(err, IsNil)
   372  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   373  		{
   374  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46},
   375  			Val:    []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0},
   376  			RowID:  70,
   377  			Offset: 1234,
   378  		},
   379  	}})
   380  	c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoRandomType).Base(), Equals, int64(70))
   381  
   382  	pairs, err = encoder.Encode(logger, []types.Datum{types.NewStringDatum("")}, 71, []int{-1, 0}, 1234)
   383  	c.Assert(err, IsNil)
   384  	c.Assert(pairs, DeepEquals, &KvPairs{pairs: []common.KvPair{
   385  		{
   386  			Key:    []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x47},
   387  			Val:    []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0},
   388  			RowID:  71,
   389  			Offset: 1234,
   390  		},
   391  	}})
   392  	c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoRandomType).Base(), Equals, int64(71))
   393  }
   394  
   395  func (s *kvSuite) TestShardRowId(c *C) {
   396  	tblInfo := mockTableInfo(c, "create table t (s varchar(16)) shard_row_id_bits = 3;")
   397  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo)
   398  	c.Assert(err, IsNil)
   399  	encoder, err := NewTableKVEncoder(tbl, &SessionOptions{
   400  		SQLMode:        mysql.ModeStrictAllTables,
   401  		Timestamp:      1234567893,
   402  		SysVars:        map[string]string{"tidb_row_format_version": "2"},
   403  		AutoRandomSeed: 456,
   404  	})
   405  	c.Assert(err, IsNil)
   406  	logger := log.Logger{Logger: zap.NewNop()}
   407  	keyMap := make(map[int64]struct{}, 16)
   408  	for i := int64(1); i <= 32; i++ {
   409  		pairs, err := encoder.Encode(logger, []types.Datum{types.NewStringDatum(fmt.Sprintf("%d", i))}, i, []int{0, -1}, i*32)
   410  		c.Assert(err, IsNil)
   411  		kvs := pairs.(*KvPairs)
   412  		c.Assert(len(kvs.pairs), Equals, 1)
   413  		_, h, err := tablecodec.DecodeRecordKey(kvs.pairs[0].Key)
   414  		c.Assert(err, IsNil)
   415  		rowID := h.IntValue()
   416  		c.Assert(rowID&((1<<60)-1), Equals, i)
   417  		keyMap[rowID>>60] = struct{}{}
   418  	}
   419  	c.Assert(len(keyMap), Equals, 8)
   420  	c.Assert(tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.RowIDAllocType).Base(), Equals, int64(32))
   421  }
   422  
   423  func (s *kvSuite) TestSplitIntoChunks(c *C) {
   424  	pairs := []common.KvPair{
   425  		{
   426  			Key:    []byte{1, 2, 3},
   427  			Val:    []byte{4, 5, 6},
   428  			Offset: 1000,
   429  		},
   430  		{
   431  			Key:    []byte{7, 8},
   432  			Val:    []byte{9, 0},
   433  			Offset: 2000,
   434  		},
   435  		{
   436  			Key:    []byte{1, 2, 3, 4},
   437  			Val:    []byte{5, 6, 7, 8},
   438  			Offset: 3000,
   439  		},
   440  		{
   441  			Key:    []byte{9, 0},
   442  			Val:    []byte{1, 2},
   443  			Offset: 4000,
   444  		},
   445  	}
   446  
   447  	splitBy10 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(10)
   448  	c.Assert(splitBy10, DeepEquals, []Rows{
   449  		MakeRowsFromKvPairs(pairs[0:2]),
   450  		MakeRowsFromKvPairs(pairs[2:3]),
   451  		MakeRowsFromKvPairs(pairs[3:4]),
   452  	})
   453  
   454  	splitBy12 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(12)
   455  	c.Assert(splitBy12, DeepEquals, []Rows{
   456  		MakeRowsFromKvPairs(pairs[0:2]),
   457  		MakeRowsFromKvPairs(pairs[2:4]),
   458  	})
   459  
   460  	splitBy1000 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(1000)
   461  	c.Assert(splitBy1000, DeepEquals, []Rows{
   462  		MakeRowsFromKvPairs(pairs[0:4]),
   463  	})
   464  
   465  	splitBy1 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(1)
   466  	c.Assert(splitBy1, DeepEquals, []Rows{
   467  		MakeRowsFromKvPairs(pairs[0:1]),
   468  		MakeRowsFromKvPairs(pairs[1:2]),
   469  		MakeRowsFromKvPairs(pairs[2:3]),
   470  		MakeRowsFromKvPairs(pairs[3:4]),
   471  	})
   472  }
   473  
   474  func (s *kvSuite) TestClassifyAndAppend(c *C) {
   475  	kvs := MakeRowFromKvPairs([]common.KvPair{
   476  		{
   477  			Key: []byte("txxxxxxxx_ryyyyyyyy"),
   478  			Val: []byte("value1"),
   479  		},
   480  		{
   481  			Key: []byte("txxxxxxxx_rwwwwwwww"),
   482  			Val: []byte("value2"),
   483  		},
   484  		{
   485  			Key: []byte("txxxxxxxx_izzzzzzzz"),
   486  			Val: []byte("index1"),
   487  		},
   488  	})
   489  
   490  	data := MakeRowsFromKvPairs(nil)
   491  	indices := MakeRowsFromKvPairs(nil)
   492  	dataChecksum := verification.MakeKVChecksum(0, 0, 0)
   493  	indexChecksum := verification.MakeKVChecksum(0, 0, 0)
   494  
   495  	kvs.ClassifyAndAppend(&data, &dataChecksum, &indices, &indexChecksum)
   496  
   497  	c.Assert(data, DeepEquals, MakeRowsFromKvPairs([]common.KvPair{
   498  		{
   499  			Key: []byte("txxxxxxxx_ryyyyyyyy"),
   500  			Val: []byte("value1"),
   501  		},
   502  		{
   503  			Key: []byte("txxxxxxxx_rwwwwwwww"),
   504  			Val: []byte("value2"),
   505  		},
   506  	}))
   507  	c.Assert(indices, DeepEquals, MakeRowsFromKvPairs([]common.KvPair{
   508  		{
   509  			Key: []byte("txxxxxxxx_izzzzzzzz"),
   510  			Val: []byte("index1"),
   511  		},
   512  	}))
   513  	c.Assert(dataChecksum.SumKVS(), Equals, uint64(2))
   514  	c.Assert(indexChecksum.SumKVS(), Equals, uint64(1))
   515  }
   516  
   517  type benchSQL2KVSuite struct {
   518  	row     []types.Datum
   519  	colPerm []int
   520  	encoder Encoder
   521  	logger  log.Logger
   522  }
   523  
   524  var _ = Suite(&benchSQL2KVSuite{})
   525  
   526  func (s *benchSQL2KVSuite) SetUpTest(c *C) {
   527  	// First, create the table info corresponding to TPC-C's "CUSTOMER" table.
   528  	p := parser.New()
   529  	se := mock.NewContext()
   530  	node, err := p.ParseOneStmt(`
   531  		create table bmsql_customer(
   532  			c_w_id         integer not null,
   533  			c_d_id         integer not null,
   534  			c_id           integer not null,
   535  			c_discount     decimal(4,4),
   536  			c_credit       char(2),
   537  			c_last         varchar(16),
   538  			c_first        varchar(16),
   539  			c_credit_lim   decimal(12,2),
   540  			c_balance      decimal(12,2),
   541  			c_ytd_payment  decimal(12,2),
   542  			c_payment_cnt  integer,
   543  			c_delivery_cnt integer,
   544  			c_street_1     varchar(20),
   545  			c_street_2     varchar(20),
   546  			c_city         varchar(20),
   547  			c_state        char(2),
   548  			c_zip          char(9),
   549  			c_phone        char(16),
   550  			c_since        timestamp,
   551  			c_middle       char(2),
   552  			c_data         varchar(500),
   553  			primary key (c_w_id, c_d_id, c_id)
   554  		);
   555  	`, "", "")
   556  	c.Assert(err, IsNil)
   557  	tableInfo, err := ddl.MockTableInfo(se, node.(*ast.CreateTableStmt), 123456)
   558  	c.Assert(err, IsNil)
   559  	tableInfo.State = model.StatePublic
   560  
   561  	// Construct the corresponding KV encoder.
   562  	tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tableInfo)
   563  	c.Assert(err, IsNil)
   564  	s.encoder, err = NewTableKVEncoder(tbl, &SessionOptions{SysVars: map[string]string{"tidb_row_format_version": "2"}})
   565  	c.Assert(err, IsNil)
   566  	s.logger = log.Logger{Logger: zap.NewNop()}
   567  
   568  	// Prepare the row to insert.
   569  	s.row = []types.Datum{
   570  		types.NewIntDatum(15),
   571  		types.NewIntDatum(10),
   572  		types.NewIntDatum(3000),
   573  		types.NewStringDatum("0.3646"),
   574  		types.NewStringDatum("GC"),
   575  		types.NewStringDatum("CALLYPRIANTI"),
   576  		types.NewStringDatum("Rg6mDFlVnP5yh"),
   577  		types.NewStringDatum("50000.0"),
   578  		types.NewStringDatum("-10.0"),
   579  		types.NewStringDatum("10.0"),
   580  		types.NewIntDatum(1),
   581  		types.NewIntDatum(0),
   582  		types.NewStringDatum("aJK7CuRnE0NUxNHSX"),
   583  		types.NewStringDatum("Q1rps77cXYoj"),
   584  		types.NewStringDatum("MigXbS6UoUS"),
   585  		types.NewStringDatum("UJ"),
   586  		types.NewStringDatum("638611111"),
   587  		types.NewStringDatum("7743262784364376"),
   588  		types.NewStringDatum("2020-02-05 19:29:58.903970"),
   589  		types.NewStringDatum("OE"),
   590  		types.NewStringDatum("H5p3dpjp7uu8n1l3j0o1buecfV6FngNNgftpNALDhOzJaSzMCMlrQwXuvLAFPIFg215D3wAYB62kiixIuasfbD729oq8TwgKzPPsx8kHE1b4AdhHwpCml3ELKiwuNGQl7CcBQOiq6aFEMMHzjGwQyXwGey0wutjp2KP3Nd4qj3FHtmHbsD8cJ0pH9TswNmdQBgXsFPZeJJhsG3rTimQpS9Tmn3vNeI9fFas3ClDZuQtBjqoTJlyzmBIYT8HeV3TuS93TNFDaXZpQqh8HsvlPq4uTTLOO9CguiY29zlSmIjkZYtva3iscG3YDOQVLeGpP9dtqEJwlRvJ4oe9jWkvRMlCeslSNEuzLxjUBtJBnGRFAzJF6RMlIWCkdCpIhcnIy3jUEsxTuiAU3hsZxUjLg2dnOG62h5qR"),
   591  	}
   592  	s.colPerm = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, -1}
   593  }
   594  
   595  // Run `go test github.com/pingcap/br/pkg/lightning/backend -check.b -test.v` to get benchmark result.
   596  func (s *benchSQL2KVSuite) BenchmarkSQL2KV(c *C) {
   597  	for i := 0; i < c.N; i++ {
   598  		rows, err := s.encoder.Encode(s.logger, s.row, 1, s.colPerm, 0)
   599  		c.Assert(err, IsNil)
   600  		c.Assert(rows, HasLen, 2)
   601  	}
   602  }