github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/dmlsink/cloudstorage/encoding_worker_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  package cloudstorage
    14  
    15  import (
    16  	"context"
    17  	"fmt"
    18  	"net/url"
    19  	"sync"
    20  	"testing"
    21  
    22  	timodel "github.com/pingcap/tidb/pkg/parser/model"
    23  	"github.com/pingcap/tidb/pkg/parser/mysql"
    24  	"github.com/pingcap/tidb/pkg/types"
    25  	"github.com/pingcap/tiflow/cdc/model"
    26  	"github.com/pingcap/tiflow/cdc/sink/dmlsink"
    27  	"github.com/pingcap/tiflow/cdc/sink/util"
    28  	"github.com/pingcap/tiflow/pkg/chann"
    29  	"github.com/pingcap/tiflow/pkg/config"
    30  	"github.com/pingcap/tiflow/pkg/sink/cloudstorage"
    31  	"github.com/pingcap/tiflow/pkg/sink/codec/builder"
    32  	"github.com/stretchr/testify/require"
    33  	"golang.org/x/sync/errgroup"
    34  )
    35  
    36  func testEncodingWorker(
    37  	t *testing.T,
    38  ) (*encodingWorker, chan eventFragment, chan eventFragment) {
    39  	uri := fmt.Sprintf("file:///%s", t.TempDir())
    40  	sinkURI, err := url.Parse(uri)
    41  	require.Nil(t, err)
    42  
    43  	changefeedID := model.DefaultChangeFeedID("changefeed-test")
    44  	encoderConfig, err := util.GetEncoderConfig(changefeedID, sinkURI, config.ProtocolCsv,
    45  		config.GetDefaultReplicaConfig(), config.DefaultMaxMessageBytes)
    46  	require.Nil(t, err)
    47  	encoderBuilder, err := builder.NewTxnEventEncoderBuilder(encoderConfig)
    48  	require.Nil(t, err)
    49  	encoder := encoderBuilder.Build()
    50  
    51  	encodedCh := make(chan eventFragment)
    52  	msgCh := make(chan eventFragment, 1024)
    53  	return newEncodingWorker(1, changefeedID, encoder, msgCh, encodedCh), msgCh, encodedCh
    54  }
    55  
    56  func TestEncodeEvents(t *testing.T) {
    57  	t.Parallel()
    58  
    59  	encodingWorker, _, encodedCh := testEncodingWorker(t)
    60  	ctx, cancel := context.WithCancel(context.Background())
    61  	eg, egCtx := errgroup.WithContext(ctx)
    62  	outputChs := []*chann.DrainableChann[eventFragment]{chann.NewAutoDrainChann[eventFragment]()}
    63  	defragmenter := newDefragmenter(encodedCh, outputChs)
    64  	eg.Go(func() error {
    65  		return defragmenter.run(egCtx)
    66  	})
    67  
    68  	tidbTableInfo := &timodel.TableInfo{
    69  		ID:   100,
    70  		Name: timodel.NewCIStr("table1"),
    71  		Columns: []*timodel.ColumnInfo{
    72  			{ID: 1, Name: timodel.NewCIStr("c1"), FieldType: *types.NewFieldType(mysql.TypeLong)},
    73  			{ID: 2, Name: timodel.NewCIStr("c2"), FieldType: *types.NewFieldType(mysql.TypeString)},
    74  		},
    75  	}
    76  	tableInfo := model.WrapTableInfo(100, "test", 33, tidbTableInfo)
    77  
    78  	err := encodingWorker.encodeEvents(eventFragment{
    79  		versionedTable: cloudstorage.VersionedTableName{
    80  			TableNameWithPhysicTableID: model.TableName{
    81  				Schema:  "test",
    82  				Table:   "table1",
    83  				TableID: 100,
    84  			},
    85  		},
    86  		seqNumber: 1,
    87  		event: &dmlsink.TxnCallbackableEvent{
    88  			Event: &model.SingleTableTxn{
    89  				TableInfo: tableInfo,
    90  				Rows: []*model.RowChangedEvent{
    91  					{
    92  						PhysicalTableID: 100,
    93  						TableInfo:       tableInfo,
    94  						Columns: []*model.ColumnData{
    95  							{ColumnID: 1, Value: 100},
    96  							{ColumnID: 2, Value: "hello world"},
    97  						},
    98  					},
    99  					{
   100  						PhysicalTableID: 100,
   101  						TableInfo:       tableInfo,
   102  						Columns: []*model.ColumnData{
   103  							{ColumnID: 1, Value: 200},
   104  							{ColumnID: 2, Value: "你好,世界"},
   105  						},
   106  					},
   107  				},
   108  			},
   109  		},
   110  	})
   111  	require.Nil(t, err)
   112  	cancel()
   113  	require.ErrorIs(t, eg.Wait(), context.Canceled)
   114  }
   115  
   116  func TestEncodingWorkerRun(t *testing.T) {
   117  	t.Parallel()
   118  
   119  	encodingWorker, msgCh, encodedCh := testEncodingWorker(t)
   120  	ctx, cancel := context.WithCancel(context.Background())
   121  	eg, egCtx := errgroup.WithContext(ctx)
   122  	outputChs := []*chann.DrainableChann[eventFragment]{chann.NewAutoDrainChann[eventFragment]()}
   123  	defragmenter := newDefragmenter(encodedCh, outputChs)
   124  	eg.Go(func() error {
   125  		return defragmenter.run(egCtx)
   126  	})
   127  
   128  	table := model.TableName{
   129  		Schema:  "test",
   130  		Table:   "table1",
   131  		TableID: 100,
   132  	}
   133  	tidbTableInfo := &timodel.TableInfo{
   134  		ID:   100,
   135  		Name: timodel.NewCIStr("table1"),
   136  		Columns: []*timodel.ColumnInfo{
   137  			{ID: 1, Name: timodel.NewCIStr("c1"), FieldType: *types.NewFieldType(mysql.TypeLong)},
   138  			{ID: 2, Name: timodel.NewCIStr("c2"), FieldType: *types.NewFieldType(mysql.TypeVarchar)},
   139  		},
   140  	}
   141  	tableInfo := model.WrapTableInfo(100, "test", 33, tidbTableInfo)
   142  	event := &model.SingleTableTxn{
   143  		TableInfo: tableInfo,
   144  		Rows: []*model.RowChangedEvent{
   145  			{
   146  				PhysicalTableID: 100,
   147  				TableInfo:       tableInfo,
   148  				Columns: []*model.ColumnData{
   149  					{ColumnID: 1, Value: 100},
   150  					{ColumnID: 2, Value: "hello world"},
   151  				},
   152  			},
   153  		},
   154  	}
   155  
   156  	for i := 0; i < 3; i++ {
   157  		frag := eventFragment{
   158  			versionedTable: cloudstorage.VersionedTableName{
   159  				TableNameWithPhysicTableID: table,
   160  			},
   161  			seqNumber: uint64(i + 1),
   162  			event: &dmlsink.TxnCallbackableEvent{
   163  				Event: event,
   164  			},
   165  		}
   166  		msgCh <- frag
   167  	}
   168  
   169  	var wg sync.WaitGroup
   170  	wg.Add(1)
   171  	go func() {
   172  		defer wg.Done()
   173  		_ = encodingWorker.run(ctx)
   174  	}()
   175  
   176  	cancel()
   177  	encodingWorker.close()
   178  	wg.Wait()
   179  }