github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/dmlsink/event_appender_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License
    13  
    14  package dmlsink
    15  
    16  import (
    17  	"testing"
    18  
    19  	"github.com/pingcap/tiflow/cdc/model"
    20  	"github.com/stretchr/testify/require"
    21  )
    22  
    23  func TestRowChangeEventAppender(t *testing.T) {
    24  	t.Parallel()
    25  
    26  	tableInfo := &model.TableInfo{
    27  		TableName: model.TableName{
    28  			Schema:      "test",
    29  			Table:       "t1",
    30  			TableID:     1,
    31  			IsPartition: false,
    32  		},
    33  		Version: 1,
    34  	}
    35  
    36  	appender := &RowChangeEventAppender{}
    37  	var buffer []*model.RowChangedEvent
    38  	rows := []*model.RowChangedEvent{
    39  		{
    40  			TableInfo: tableInfo,
    41  			CommitTs:  1,
    42  		},
    43  		{
    44  			TableInfo: tableInfo,
    45  			CommitTs:  2,
    46  		},
    47  		{
    48  			TableInfo: tableInfo,
    49  			CommitTs:  2,
    50  		},
    51  	}
    52  	buffer = appender.Append(buffer, rows...)
    53  	require.Len(t, buffer, 3)
    54  	// Should be ordered by commitTs.
    55  	require.Equal(t, uint64(1), buffer[0].GetCommitTs())
    56  	require.Equal(t, uint64(2), buffer[1].GetCommitTs())
    57  	require.Equal(t, uint64(2), buffer[2].GetCommitTs())
    58  }
    59  
    60  func TestTxnEventAppenderWithoutIgnoreStartTs(t *testing.T) {
    61  	t.Parallel()
    62  
    63  	tableInfo := &model.TableInfo{
    64  		Version: 1,
    65  		TableName: model.TableName{
    66  			Schema:      "test",
    67  			Table:       "t1",
    68  			TableID:     1,
    69  			IsPartition: false,
    70  		},
    71  	}
    72  
    73  	appender := &TxnEventAppender{}
    74  	var buffer []*model.SingleTableTxn
    75  	rows := []*model.RowChangedEvent{
    76  		{
    77  			TableInfo: tableInfo,
    78  			CommitTs:  101,
    79  			StartTs:   98,
    80  		},
    81  		{
    82  			TableInfo: tableInfo,
    83  			CommitTs:  102,
    84  			StartTs:   99,
    85  		},
    86  		{
    87  			TableInfo: tableInfo,
    88  			CommitTs:  102,
    89  			StartTs:   100,
    90  		},
    91  		{
    92  			TableInfo: tableInfo,
    93  			CommitTs:  102,
    94  			StartTs:   100,
    95  		},
    96  		{
    97  			TableInfo: tableInfo,
    98  			CommitTs:  103,
    99  			StartTs:   101,
   100  		},
   101  		{
   102  			TableInfo: tableInfo,
   103  			CommitTs:  103,
   104  			StartTs:   101,
   105  		},
   106  		{
   107  			TableInfo: tableInfo,
   108  			CommitTs:  104,
   109  			StartTs:   102,
   110  		},
   111  		{
   112  			TableInfo: tableInfo,
   113  			CommitTs:  105,
   114  			StartTs:   103,
   115  			// Batch1
   116  			SplitTxn: true,
   117  		},
   118  		{
   119  			TableInfo: tableInfo,
   120  			CommitTs:  105,
   121  			StartTs:   103,
   122  		},
   123  		{
   124  			TableInfo: tableInfo,
   125  			CommitTs:  105,
   126  			StartTs:   103,
   127  		},
   128  		{
   129  			TableInfo: tableInfo,
   130  			CommitTs:  105,
   131  			StartTs:   103,
   132  			// Batch2
   133  			SplitTxn: true,
   134  		},
   135  		{
   136  			TableInfo: tableInfo,
   137  			CommitTs:  105,
   138  			StartTs:   103,
   139  		},
   140  	}
   141  	buffer = appender.Append(buffer, rows...)
   142  	require.Len(t, buffer, 7)
   143  	// Make sure the order is correct.
   144  	require.Equal(t, uint64(101), buffer[0].GetCommitTs())
   145  	// Make sure grouped by startTs and batch.
   146  	require.Len(t, buffer[0].Rows, 1)
   147  
   148  	require.Equal(t, uint64(102), buffer[1].GetCommitTs())
   149  	require.Len(t, buffer[1].Rows, 1)
   150  
   151  	require.Equal(t, uint64(102), buffer[2].GetCommitTs())
   152  	require.Len(t, buffer[2].Rows, 2)
   153  
   154  	require.Equal(t, uint64(103), buffer[3].GetCommitTs())
   155  	require.Len(t, buffer[3].Rows, 2)
   156  
   157  	require.Equal(t, uint64(104), buffer[4].GetCommitTs())
   158  	require.Len(t, buffer[4].Rows, 1)
   159  
   160  	require.Equal(t, uint64(105), buffer[5].GetCommitTs())
   161  	require.Len(t, buffer[5].Rows, 3)
   162  
   163  	require.Equal(t, uint64(105), buffer[6].GetCommitTs())
   164  	require.Len(t, buffer[6].Rows, 2)
   165  
   166  	// Test the case which the commitTs is not strictly increasing.
   167  	rows = []*model.RowChangedEvent{
   168  		{
   169  			TableInfo: tableInfo,
   170  			CommitTs:  101,
   171  			StartTs:   98,
   172  		},
   173  		{
   174  			TableInfo: tableInfo,
   175  			CommitTs:  100,
   176  			StartTs:   99,
   177  		},
   178  	}
   179  	buffer = buffer[:0]
   180  	require.Panics(t, func() {
   181  		buffer = appender.Append(buffer, rows...)
   182  	})
   183  
   184  	// Test the case which the startTs is not strictly increasing.
   185  	rows = []*model.RowChangedEvent{
   186  		{
   187  			TableInfo: tableInfo,
   188  			CommitTs:  101,
   189  			StartTs:   98,
   190  		},
   191  		{
   192  			TableInfo: tableInfo,
   193  			CommitTs:  101,
   194  			StartTs:   80,
   195  		},
   196  	}
   197  	buffer = buffer[:0]
   198  	require.Panics(t, func() {
   199  		buffer = appender.Append(buffer, rows...)
   200  	})
   201  }
   202  
   203  func TestTxnEventAppenderWithIgnoreStartTs(t *testing.T) {
   204  	t.Parallel()
   205  
   206  	tableInfo := &model.TableInfo{
   207  		Version: 1,
   208  		TableName: model.TableName{
   209  			Schema:      "test",
   210  			Table:       "t1",
   211  			TableID:     1,
   212  			IsPartition: false,
   213  		},
   214  	}
   215  
   216  	appender := &TxnEventAppender{IgnoreStartTs: true}
   217  	var buffer []*model.SingleTableTxn
   218  	rows := []*model.RowChangedEvent{
   219  		{
   220  			TableInfo: tableInfo,
   221  			CommitTs:  101,
   222  			StartTs:   0,
   223  		},
   224  		{
   225  			TableInfo: tableInfo,
   226  			CommitTs:  101,
   227  			StartTs:   0,
   228  		},
   229  		{
   230  			TableInfo: tableInfo,
   231  			CommitTs:  102,
   232  			StartTs:   90,
   233  		},
   234  		{
   235  			TableInfo: tableInfo,
   236  			CommitTs:  102,
   237  			StartTs:   91,
   238  		},
   239  		{
   240  			TableInfo: tableInfo,
   241  			CommitTs:  103,
   242  			StartTs:   0,
   243  		},
   244  		{
   245  			TableInfo: tableInfo,
   246  			CommitTs:  103,
   247  			StartTs:   0,
   248  		},
   249  		{
   250  			TableInfo: tableInfo,
   251  			CommitTs:  104,
   252  			StartTs:   0,
   253  		},
   254  		{
   255  			TableInfo: tableInfo,
   256  			CommitTs:  105,
   257  			StartTs:   0,
   258  			// Batch1
   259  			SplitTxn: true,
   260  		},
   261  		{
   262  			TableInfo: tableInfo,
   263  			CommitTs:  105,
   264  			StartTs:   0,
   265  		},
   266  		{
   267  			TableInfo: tableInfo,
   268  			CommitTs:  105,
   269  			StartTs:   0,
   270  		},
   271  		{
   272  			TableInfo: tableInfo,
   273  			CommitTs:  105,
   274  			StartTs:   0,
   275  			// Batch2
   276  			SplitTxn: true,
   277  		},
   278  		{
   279  			TableInfo: tableInfo,
   280  			CommitTs:  105,
   281  			StartTs:   0,
   282  		},
   283  	}
   284  	buffer = appender.Append(buffer, rows...)
   285  	require.Len(t, buffer, 7)
   286  	// Make sure the order is correct.
   287  	require.Equal(t, uint64(101), buffer[0].GetCommitTs())
   288  	// Make we can ignore the startTs.
   289  	require.Len(t, buffer[0].Rows, 2)
   290  
   291  	// Make sure if the startTs is not 0, we can't deal with it.
   292  	require.Equal(t, uint64(102), buffer[1].GetCommitTs())
   293  	require.Len(t, buffer[1].Rows, 1)
   294  	require.Equal(t, uint64(102), buffer[2].GetCommitTs())
   295  	require.Len(t, buffer[2].Rows, 1)
   296  
   297  	require.Equal(t, uint64(103), buffer[3].GetCommitTs())
   298  	require.Len(t, buffer[3].Rows, 2)
   299  
   300  	require.Equal(t, uint64(104), buffer[4].GetCommitTs())
   301  	require.Len(t, buffer[4].Rows, 1)
   302  
   303  	// First batch.
   304  	require.Equal(t, uint64(105), buffer[5].GetCommitTs())
   305  	require.Len(t, buffer[5].Rows, 3)
   306  
   307  	// Second batch.
   308  	require.Equal(t, uint64(105), buffer[6].GetCommitTs())
   309  	require.Len(t, buffer[6].Rows, 2)
   310  
   311  	// Test the case which the commitTs is not strictly increasing.
   312  	rows = []*model.RowChangedEvent{
   313  		{
   314  			TableInfo: tableInfo,
   315  			CommitTs:  101,
   316  			StartTs:   98,
   317  		},
   318  		{
   319  			TableInfo: tableInfo,
   320  			CommitTs:  100,
   321  			StartTs:   99,
   322  		},
   323  	}
   324  	buffer = buffer[:0]
   325  	require.Panics(t, func() {
   326  		buffer = appender.Append(buffer, rows...)
   327  	})
   328  
   329  	// Test the case which the startTs is not strictly increasing.
   330  	rows = []*model.RowChangedEvent{
   331  		{
   332  			TableInfo: tableInfo,
   333  			CommitTs:  101,
   334  			StartTs:   98,
   335  		},
   336  		{
   337  			TableInfo: tableInfo,
   338  			CommitTs:  101,
   339  			StartTs:   80,
   340  		},
   341  	}
   342  	buffer = buffer[:0]
   343  	require.Panics(t, func() {
   344  		buffer = appender.Append(buffer, rows...)
   345  	})
   346  
   347  	// Test the case which the startTs all is 0.
   348  	rows = []*model.RowChangedEvent{
   349  		{
   350  			TableInfo: tableInfo,
   351  			CommitTs:  101,
   352  			StartTs:   0,
   353  		},
   354  		{
   355  			TableInfo: tableInfo,
   356  			CommitTs:  101,
   357  			StartTs:   0,
   358  		},
   359  	}
   360  	buffer = buffer[:0]
   361  	require.NotPanics(t, func() {
   362  		buffer = appender.Append(buffer, rows...)
   363  	})
   364  }