github.com/pingcap/tidb-lightning@v5.0.0-rc.0.20210428090220-84b649866577+incompatible/lightning/checkpoints/checkpoints_test.go (about)

     1  package checkpoints
     2  
     3  import (
     4  	"path/filepath"
     5  	"testing"
     6  
     7  	. "github.com/pingcap/check"
     8  
     9  	"github.com/pingcap/tidb-lightning/lightning/mydump"
    10  	"github.com/pingcap/tidb-lightning/lightning/verification"
    11  )
    12  
    13  func Test(t *testing.T) {
    14  	TestingT(t)
    15  }
    16  
    17  var _ = Suite(&checkpointSuite{})
    18  
    19  type checkpointSuite struct {
    20  }
    21  
    22  func (s *checkpointSuite) TestMergeStatusCheckpoint(c *C) {
    23  	cpd := NewTableCheckpointDiff()
    24  
    25  	m := StatusCheckpointMerger{EngineID: 0, Status: CheckpointStatusImported}
    26  	m.MergeInto(cpd)
    27  
    28  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
    29  		hasStatus: false,
    30  		engines: map[int32]engineCheckpointDiff{
    31  			0: {
    32  				hasStatus: true,
    33  				status:    CheckpointStatusImported,
    34  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    35  			},
    36  		},
    37  	})
    38  
    39  	m = StatusCheckpointMerger{EngineID: -1, Status: CheckpointStatusLoaded}
    40  	m.MergeInto(cpd)
    41  
    42  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
    43  		hasStatus: false,
    44  		engines: map[int32]engineCheckpointDiff{
    45  			0: {
    46  				hasStatus: true,
    47  				status:    CheckpointStatusImported,
    48  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    49  			},
    50  			-1: {
    51  				hasStatus: true,
    52  				status:    CheckpointStatusLoaded,
    53  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    54  			},
    55  		},
    56  	})
    57  
    58  	m = StatusCheckpointMerger{EngineID: WholeTableEngineID, Status: CheckpointStatusClosed}
    59  	m.MergeInto(cpd)
    60  
    61  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
    62  		hasStatus: true,
    63  		status:    CheckpointStatusClosed,
    64  		engines: map[int32]engineCheckpointDiff{
    65  			0: {
    66  				hasStatus: true,
    67  				status:    CheckpointStatusImported,
    68  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    69  			},
    70  			-1: {
    71  				hasStatus: true,
    72  				status:    CheckpointStatusLoaded,
    73  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    74  			},
    75  		},
    76  	})
    77  
    78  	m = StatusCheckpointMerger{EngineID: -1, Status: CheckpointStatusAllWritten}
    79  	m.MergeInto(cpd)
    80  
    81  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
    82  		hasStatus: true,
    83  		status:    CheckpointStatusClosed,
    84  		engines: map[int32]engineCheckpointDiff{
    85  			0: {
    86  				hasStatus: true,
    87  				status:    CheckpointStatusImported,
    88  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    89  			},
    90  			-1: {
    91  				hasStatus: true,
    92  				status:    CheckpointStatusAllWritten,
    93  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
    94  			},
    95  		},
    96  	})
    97  }
    98  
    99  func (s *checkpointSuite) TestMergeInvalidStatusCheckpoint(c *C) {
   100  	cpd := NewTableCheckpointDiff()
   101  
   102  	m := StatusCheckpointMerger{EngineID: 0, Status: CheckpointStatusLoaded}
   103  	m.MergeInto(cpd)
   104  
   105  	m = StatusCheckpointMerger{EngineID: -1, Status: CheckpointStatusAllWritten}
   106  	m.SetInvalid()
   107  	m.MergeInto(cpd)
   108  
   109  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
   110  		hasStatus: true,
   111  		status:    CheckpointStatusAllWritten / 10,
   112  		engines: map[int32]engineCheckpointDiff{
   113  			0: {
   114  				hasStatus: true,
   115  				status:    CheckpointStatusLoaded,
   116  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
   117  			},
   118  			-1: {
   119  				hasStatus: true,
   120  				status:    CheckpointStatusAllWritten / 10,
   121  				chunks:    make(map[ChunkCheckpointKey]chunkCheckpointDiff),
   122  			},
   123  		},
   124  	})
   125  }
   126  
   127  func (s *checkpointSuite) TestMergeChunkCheckpoint(c *C) {
   128  	cpd := NewTableCheckpointDiff()
   129  
   130  	key := ChunkCheckpointKey{Path: "/tmp/path/1.sql", Offset: 0}
   131  
   132  	m := ChunkCheckpointMerger{
   133  		EngineID: 2,
   134  		Key:      key,
   135  		Checksum: verification.MakeKVChecksum(700, 15, 1234567890),
   136  		Pos:      1055,
   137  		RowID:    31,
   138  	}
   139  	m.MergeInto(cpd)
   140  
   141  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
   142  		engines: map[int32]engineCheckpointDiff{
   143  			2: {
   144  				chunks: map[ChunkCheckpointKey]chunkCheckpointDiff{
   145  					key: {
   146  						pos:      1055,
   147  						rowID:    31,
   148  						checksum: verification.MakeKVChecksum(700, 15, 1234567890),
   149  					},
   150  				},
   151  			},
   152  		},
   153  	})
   154  
   155  	m = ChunkCheckpointMerger{
   156  		EngineID: 2,
   157  		Key:      key,
   158  		Checksum: verification.MakeKVChecksum(800, 20, 1357924680),
   159  		Pos:      1080,
   160  		RowID:    42,
   161  	}
   162  	m.MergeInto(cpd)
   163  
   164  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
   165  		engines: map[int32]engineCheckpointDiff{
   166  			2: {
   167  				chunks: map[ChunkCheckpointKey]chunkCheckpointDiff{
   168  					key: {
   169  						pos:      1080,
   170  						rowID:    42,
   171  						checksum: verification.MakeKVChecksum(800, 20, 1357924680),
   172  					},
   173  				},
   174  			},
   175  		},
   176  	})
   177  }
   178  
   179  func (s *checkpointSuite) TestRebaseCheckpoint(c *C) {
   180  	cpd := NewTableCheckpointDiff()
   181  
   182  	m := RebaseCheckpointMerger{AllocBase: 10000}
   183  	m.MergeInto(cpd)
   184  
   185  	c.Assert(cpd, DeepEquals, &TableCheckpointDiff{
   186  		hasRebase: true,
   187  		allocBase: 10000,
   188  		engines:   make(map[int32]engineCheckpointDiff),
   189  	})
   190  }
   191  
   192  func (s *checkpointSuite) TestApplyDiff(c *C) {
   193  	cp := TableCheckpoint{
   194  		Status:    CheckpointStatusLoaded,
   195  		AllocBase: 123,
   196  		Engines: map[int32]*EngineCheckpoint{
   197  			-1: {
   198  				Status: CheckpointStatusLoaded,
   199  			},
   200  			0: {
   201  				Status: CheckpointStatusLoaded,
   202  				Chunks: []*ChunkCheckpoint{
   203  					{
   204  						Key: ChunkCheckpointKey{Path: "/tmp/01.sql"},
   205  						Chunk: mydump.Chunk{
   206  							Offset:       0,
   207  							EndOffset:    20000,
   208  							PrevRowIDMax: 0,
   209  							RowIDMax:     1000,
   210  						},
   211  					},
   212  					{
   213  						Key: ChunkCheckpointKey{Path: "/tmp/04.sql"},
   214  						Chunk: mydump.Chunk{
   215  							Offset:       0,
   216  							EndOffset:    15000,
   217  							PrevRowIDMax: 1000,
   218  							RowIDMax:     1300,
   219  						},
   220  					},
   221  				},
   222  			},
   223  		},
   224  	}
   225  
   226  	cpd := NewTableCheckpointDiff()
   227  	(&StatusCheckpointMerger{EngineID: -1, Status: CheckpointStatusImported}).MergeInto(cpd)
   228  	(&StatusCheckpointMerger{EngineID: WholeTableEngineID, Status: CheckpointStatusAllWritten}).MergeInto(cpd)
   229  	(&StatusCheckpointMerger{EngineID: 1234, Status: CheckpointStatusAnalyzeSkipped}).MergeInto(cpd)
   230  	(&RebaseCheckpointMerger{AllocBase: 11111}).MergeInto(cpd)
   231  	(&ChunkCheckpointMerger{
   232  		EngineID: 0,
   233  		Key:      ChunkCheckpointKey{Path: "/tmp/01.sql"},
   234  		Checksum: verification.MakeKVChecksum(3333, 4444, 5555),
   235  		Pos:      6666,
   236  		RowID:    777,
   237  	}).MergeInto(cpd)
   238  	(&ChunkCheckpointMerger{
   239  		EngineID: 5678,
   240  		Key:      ChunkCheckpointKey{Path: "/tmp/04.sql"},
   241  		Pos:      9999,
   242  		RowID:    888,
   243  	}).MergeInto(cpd)
   244  	(&ChunkCheckpointMerger{
   245  		EngineID: 0,
   246  		Key:      ChunkCheckpointKey{Path: "/tmp/03.sql"},
   247  		Pos:      3636,
   248  		RowID:    2222,
   249  	}).MergeInto(cpd)
   250  	(&ChunkCheckpointMerger{
   251  		EngineID: 0,
   252  		Key:      ChunkCheckpointKey{Path: "/tmp/10.sql"},
   253  		Pos:      4949,
   254  		RowID:    444,
   255  	}).MergeInto(cpd)
   256  
   257  	cp.Apply(cpd)
   258  
   259  	c.Assert(cp, DeepEquals, TableCheckpoint{
   260  		Status:    CheckpointStatusAllWritten,
   261  		AllocBase: 11111,
   262  		Engines: map[int32]*EngineCheckpoint{
   263  			-1: {
   264  				Status: CheckpointStatusImported,
   265  			},
   266  			0: {
   267  				Status: CheckpointStatusLoaded,
   268  				Chunks: []*ChunkCheckpoint{
   269  					{
   270  						Key: ChunkCheckpointKey{Path: "/tmp/01.sql"},
   271  						Chunk: mydump.Chunk{
   272  							Offset:       6666,
   273  							EndOffset:    20000,
   274  							PrevRowIDMax: 777,
   275  							RowIDMax:     1000,
   276  						},
   277  						Checksum: verification.MakeKVChecksum(3333, 4444, 5555),
   278  					},
   279  					{
   280  						Key: ChunkCheckpointKey{Path: "/tmp/04.sql"},
   281  						Chunk: mydump.Chunk{
   282  							Offset:       0,
   283  							EndOffset:    15000,
   284  							PrevRowIDMax: 1000,
   285  							RowIDMax:     1300,
   286  						},
   287  					},
   288  				},
   289  			},
   290  		},
   291  	})
   292  }
   293  
   294  func (s *checkpointSuite) TestCheckpointMarshallUnmarshall(c *C) {
   295  	path := filepath.Join(c.MkDir(), "filecheckpoint")
   296  	fileChkp := NewFileCheckpointsDB(path)
   297  	fileChkp.checkpoints.Checkpoints["a"] = &TableCheckpointModel{
   298  		Status:  uint32(CheckpointStatusLoaded),
   299  		Engines: map[int32]*EngineCheckpointModel{},
   300  	}
   301  	fileChkp.Close()
   302  
   303  	fileChkp2 := NewFileCheckpointsDB(path)
   304  	// if not recover empty map explicitly, it will become nil
   305  	c.Assert(fileChkp2.checkpoints.Checkpoints["a"].Engines, NotNil)
   306  }