github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/dmlsink/cloudstorage/dml_worker_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  package cloudstorage
    14  
    15  import (
    16  	"context"
    17  	"fmt"
    18  	"net/url"
    19  	"path"
    20  	"sync"
    21  	"testing"
    22  	"time"
    23  
    24  	timodel "github.com/pingcap/tidb/pkg/parser/model"
    25  	"github.com/pingcap/tidb/pkg/parser/mysql"
    26  	"github.com/pingcap/tidb/pkg/parser/types"
    27  	"github.com/pingcap/tiflow/cdc/model"
    28  	"github.com/pingcap/tiflow/cdc/sink/dmlsink"
    29  	"github.com/pingcap/tiflow/cdc/sink/metrics"
    30  	"github.com/pingcap/tiflow/engine/pkg/clock"
    31  	"github.com/pingcap/tiflow/pkg/chann"
    32  	"github.com/pingcap/tiflow/pkg/config"
    33  	"github.com/pingcap/tiflow/pkg/pdutil"
    34  	"github.com/pingcap/tiflow/pkg/sink"
    35  	"github.com/pingcap/tiflow/pkg/sink/cloudstorage"
    36  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    37  	"github.com/pingcap/tiflow/pkg/util"
    38  	"github.com/stretchr/testify/require"
    39  )
    40  
    41  func testDMLWorker(ctx context.Context, t *testing.T, dir string) *dmlWorker {
    42  	uri := fmt.Sprintf("file:///%s?flush-interval=2s", dir)
    43  	storage, err := util.GetExternalStorageFromURI(ctx, uri)
    44  	require.Nil(t, err)
    45  	sinkURI, err := url.Parse(uri)
    46  	require.Nil(t, err)
    47  	cfg := cloudstorage.NewConfig()
    48  	replicaConfig := config.GetDefaultReplicaConfig()
    49  	replicaConfig.Sink.DateSeparator = util.AddressOf(config.DateSeparatorNone.String())
    50  	err = cfg.Apply(context.TODO(), sinkURI, replicaConfig)
    51  	cfg.FileIndexWidth = 6
    52  	require.Nil(t, err)
    53  
    54  	statistics := metrics.NewStatistics(ctx, model.DefaultChangeFeedID("dml-worker-test"),
    55  		sink.TxnSink)
    56  	pdlock := pdutil.NewMonotonicClock(clock.New())
    57  	d := newDMLWorker(1, model.DefaultChangeFeedID("dml-worker-test"), storage,
    58  		cfg, ".json", chann.NewAutoDrainChann[eventFragment](), pdlock, statistics)
    59  	return d
    60  }
    61  
    62  func TestDMLWorkerRun(t *testing.T) {
    63  	t.Parallel()
    64  
    65  	ctx, cancel := context.WithCancel(context.Background())
    66  	parentDir := t.TempDir()
    67  	d := testDMLWorker(ctx, t, parentDir)
    68  	fragCh := d.inputCh
    69  	table1Dir := path.Join(parentDir, "test/table1/99")
    70  	// assume table1 and table2 are dispatched to the same DML worker
    71  	table1 := model.TableName{
    72  		Schema:  "test",
    73  		Table:   "table1",
    74  		TableID: 100,
    75  	}
    76  	tidbTableInfo := &timodel.TableInfo{
    77  		ID:   100,
    78  		Name: timodel.NewCIStr("table1"),
    79  		Columns: []*timodel.ColumnInfo{
    80  			{ID: 1, Name: timodel.NewCIStr("c1"), FieldType: *types.NewFieldType(mysql.TypeLong)},
    81  			{ID: 2, Name: timodel.NewCIStr("c2"), FieldType: *types.NewFieldType(mysql.TypeVarchar)},
    82  		},
    83  	}
    84  	tableInfo := model.WrapTableInfo(100, "test", 99, tidbTableInfo)
    85  	for i := 0; i < 5; i++ {
    86  		frag := eventFragment{
    87  			seqNumber: uint64(i),
    88  			versionedTable: cloudstorage.VersionedTableName{
    89  				TableNameWithPhysicTableID: table1,
    90  				TableInfoVersion:           99,
    91  			},
    92  			event: &dmlsink.TxnCallbackableEvent{
    93  				Event: &model.SingleTableTxn{
    94  					TableInfo: tableInfo,
    95  					Rows: []*model.RowChangedEvent{
    96  						{
    97  							PhysicalTableID: 100,
    98  							TableInfo:       tableInfo,
    99  							Columns: []*model.ColumnData{
   100  								{ColumnID: 1, Value: 100},
   101  								{ColumnID: 2, Value: "hello world"},
   102  							},
   103  						},
   104  					},
   105  				},
   106  			},
   107  			encodedMsgs: []*common.Message{
   108  				{
   109  					Value: []byte(fmt.Sprintf(`{"id":%d,"database":"test","table":"table1","pkNames":[],"isDdl":false,`+
   110  						`"type":"INSERT","es":0,"ts":1663572946034,"sql":"","sqlType":{"c1":12,"c2":12},`+
   111  						`"data":[{"c1":"100","c2":"hello world"}],"old":null}`, i)),
   112  				},
   113  			},
   114  		}
   115  		fragCh.In() <- frag
   116  	}
   117  
   118  	var wg sync.WaitGroup
   119  	wg.Add(1)
   120  	go func() {
   121  		defer wg.Done()
   122  		_ = d.run(ctx)
   123  	}()
   124  
   125  	time.Sleep(4 * time.Second)
   126  	// check whether files for table1 has been generated
   127  	fileNames := getTableFiles(t, table1Dir)
   128  	require.Len(t, fileNames, 2)
   129  	require.ElementsMatch(t, []string{"CDC000001.json", "CDC.index"}, fileNames)
   130  	cancel()
   131  	d.close()
   132  	wg.Wait()
   133  	fragCh.CloseAndDrain()
   134  }