github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/dmlsink/cloudstorage/defragmenter_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  package cloudstorage
    14  
    15  import (
    16  	"context"
    17  	"math/rand"
    18  	"net/url"
    19  	"strconv"
    20  	"testing"
    21  	"time"
    22  
    23  	timodel "github.com/pingcap/tidb/pkg/parser/model"
    24  	"github.com/pingcap/tidb/pkg/parser/mysql"
    25  	"github.com/pingcap/tidb/pkg/types"
    26  	"github.com/pingcap/tiflow/cdc/model"
    27  	"github.com/pingcap/tiflow/cdc/sink/dmlsink"
    28  	"github.com/pingcap/tiflow/cdc/sink/util"
    29  	"github.com/pingcap/tiflow/pkg/chann"
    30  	"github.com/pingcap/tiflow/pkg/config"
    31  	"github.com/pingcap/tiflow/pkg/sink/cloudstorage"
    32  	"github.com/pingcap/tiflow/pkg/sink/codec/builder"
    33  	"github.com/stretchr/testify/require"
    34  	"golang.org/x/sync/errgroup"
    35  )
    36  
    37  func TestDeframenter(t *testing.T) {
    38  	t.Parallel()
    39  
    40  	ctx, cancel := context.WithCancel(context.Background())
    41  	eg, egCtx := errgroup.WithContext(ctx)
    42  
    43  	inputCh := make(chan eventFragment)
    44  	outputCh := chann.NewAutoDrainChann[eventFragment]()
    45  	defrag := newDefragmenter(inputCh, []*chann.DrainableChann[eventFragment]{outputCh})
    46  	eg.Go(func() error {
    47  		return defrag.run(egCtx)
    48  	})
    49  
    50  	uri := "file:///tmp/test"
    51  	txnCnt := 50
    52  	sinkURI, err := url.Parse(uri)
    53  	require.Nil(t, err)
    54  
    55  	changefeedID := model.DefaultChangeFeedID("changefeed-test")
    56  	encoderConfig, err := util.GetEncoderConfig(changefeedID, sinkURI, config.ProtocolCsv,
    57  		config.GetDefaultReplicaConfig(), config.DefaultMaxMessageBytes)
    58  	require.Nil(t, err)
    59  	encoderBuilder, err := builder.NewTxnEventEncoderBuilder(encoderConfig)
    60  	require.Nil(t, err)
    61  
    62  	var seqNumbers []uint64
    63  	for i := 0; i < txnCnt; i++ {
    64  		seqNumbers = append(seqNumbers, uint64(i+1))
    65  	}
    66  	rand.Seed(time.Now().UnixNano())
    67  	rand.Shuffle(len(seqNumbers), func(i, j int) {
    68  		seqNumbers[i], seqNumbers[j] = seqNumbers[j], seqNumbers[i]
    69  	})
    70  
    71  	tidbTableInfo := &timodel.TableInfo{
    72  		ID:   100,
    73  		Name: timodel.NewCIStr("table1"),
    74  		Columns: []*timodel.ColumnInfo{
    75  			{ID: 1, Name: timodel.NewCIStr("c1"), FieldType: *types.NewFieldType(mysql.TypeLong)},
    76  			{ID: 2, Name: timodel.NewCIStr("c2"), FieldType: *types.NewFieldType(mysql.TypeString)},
    77  		},
    78  	}
    79  	tableInfo := model.WrapTableInfo(100, "test", 99, tidbTableInfo)
    80  	for i := 0; i < txnCnt; i++ {
    81  		go func(seq uint64) {
    82  			encoder := encoderBuilder.Build()
    83  			frag := eventFragment{
    84  				versionedTable: cloudstorage.VersionedTableName{
    85  					TableNameWithPhysicTableID: model.TableName{
    86  						Schema:  "test",
    87  						Table:   "table1",
    88  						TableID: 100,
    89  					},
    90  				},
    91  				seqNumber: seq,
    92  				event: &dmlsink.TxnCallbackableEvent{
    93  					Event: &model.SingleTableTxn{},
    94  				},
    95  			}
    96  
    97  			rand.Seed(time.Now().UnixNano())
    98  			n := 1 + rand.Intn(1000)
    99  			for j := 0; j < n; j++ {
   100  				row := &model.RowChangedEvent{
   101  					PhysicalTableID: 100,
   102  					TableInfo:       tableInfo,
   103  					Columns: []*model.ColumnData{
   104  						{ColumnID: 1, Value: j + 1},
   105  						{ColumnID: 2, Value: "hello world"},
   106  					},
   107  				}
   108  				frag.event.Event.Rows = append(frag.event.Event.Rows, row)
   109  			}
   110  			err := encoder.AppendTxnEvent(frag.event.Event, nil)
   111  			require.NoError(t, err)
   112  			frag.encodedMsgs = encoder.Build()
   113  
   114  			for _, msg := range frag.encodedMsgs {
   115  				msg.Key = []byte(strconv.Itoa(int(seq)))
   116  			}
   117  			inputCh <- frag
   118  		}(uint64(i + 1))
   119  	}
   120  
   121  	prevSeq := 0
   122  LOOP:
   123  	for {
   124  		select {
   125  		case frag := <-outputCh.Out():
   126  			for _, msg := range frag.encodedMsgs {
   127  				curSeq, err := strconv.Atoi(string(msg.Key))
   128  				require.Nil(t, err)
   129  				require.GreaterOrEqual(t, curSeq, prevSeq)
   130  				prevSeq = curSeq
   131  			}
   132  		case <-time.After(5 * time.Second):
   133  			break LOOP
   134  		}
   135  	}
   136  	cancel()
   137  	require.ErrorIs(t, eg.Wait(), context.Canceled)
   138  }