github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/dmlsink/cloudstorage/encoding_worker_test.go (about) 1 // Copyright 2022 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 package cloudstorage 14 15 import ( 16 "context" 17 "fmt" 18 "net/url" 19 "sync" 20 "testing" 21 22 timodel "github.com/pingcap/tidb/pkg/parser/model" 23 "github.com/pingcap/tidb/pkg/parser/mysql" 24 "github.com/pingcap/tidb/pkg/types" 25 "github.com/pingcap/tiflow/cdc/model" 26 "github.com/pingcap/tiflow/cdc/sink/dmlsink" 27 "github.com/pingcap/tiflow/cdc/sink/util" 28 "github.com/pingcap/tiflow/pkg/chann" 29 "github.com/pingcap/tiflow/pkg/config" 30 "github.com/pingcap/tiflow/pkg/sink/cloudstorage" 31 "github.com/pingcap/tiflow/pkg/sink/codec/builder" 32 "github.com/stretchr/testify/require" 33 "golang.org/x/sync/errgroup" 34 ) 35 36 func testEncodingWorker( 37 t *testing.T, 38 ) (*encodingWorker, chan eventFragment, chan eventFragment) { 39 uri := fmt.Sprintf("file:///%s", t.TempDir()) 40 sinkURI, err := url.Parse(uri) 41 require.Nil(t, err) 42 43 changefeedID := model.DefaultChangeFeedID("changefeed-test") 44 encoderConfig, err := util.GetEncoderConfig(changefeedID, sinkURI, config.ProtocolCsv, 45 config.GetDefaultReplicaConfig(), config.DefaultMaxMessageBytes) 46 require.Nil(t, err) 47 encoderBuilder, err := builder.NewTxnEventEncoderBuilder(encoderConfig) 48 require.Nil(t, err) 49 encoder := encoderBuilder.Build() 50 51 encodedCh := make(chan eventFragment) 52 msgCh := make(chan eventFragment, 1024) 53 return newEncodingWorker(1, changefeedID, encoder, msgCh, encodedCh), msgCh, encodedCh 54 } 55 56 func TestEncodeEvents(t *testing.T) { 57 t.Parallel() 58 59 encodingWorker, _, encodedCh := testEncodingWorker(t) 60 ctx, cancel := context.WithCancel(context.Background()) 61 eg, egCtx := errgroup.WithContext(ctx) 62 outputChs := []*chann.DrainableChann[eventFragment]{chann.NewAutoDrainChann[eventFragment]()} 63 defragmenter := newDefragmenter(encodedCh, outputChs) 64 eg.Go(func() error { 65 return defragmenter.run(egCtx) 66 }) 67 68 tidbTableInfo := &timodel.TableInfo{ 69 ID: 100, 70 Name: timodel.NewCIStr("table1"), 71 Columns: []*timodel.ColumnInfo{ 72 {ID: 1, Name: timodel.NewCIStr("c1"), FieldType: *types.NewFieldType(mysql.TypeLong)}, 73 {ID: 2, Name: timodel.NewCIStr("c2"), FieldType: *types.NewFieldType(mysql.TypeString)}, 74 }, 75 } 76 tableInfo := model.WrapTableInfo(100, "test", 33, tidbTableInfo) 77 78 err := encodingWorker.encodeEvents(eventFragment{ 79 versionedTable: cloudstorage.VersionedTableName{ 80 TableNameWithPhysicTableID: model.TableName{ 81 Schema: "test", 82 Table: "table1", 83 TableID: 100, 84 }, 85 }, 86 seqNumber: 1, 87 event: &dmlsink.TxnCallbackableEvent{ 88 Event: &model.SingleTableTxn{ 89 TableInfo: tableInfo, 90 Rows: []*model.RowChangedEvent{ 91 { 92 PhysicalTableID: 100, 93 TableInfo: tableInfo, 94 Columns: []*model.ColumnData{ 95 {ColumnID: 1, Value: 100}, 96 {ColumnID: 2, Value: "hello world"}, 97 }, 98 }, 99 { 100 PhysicalTableID: 100, 101 TableInfo: tableInfo, 102 Columns: []*model.ColumnData{ 103 {ColumnID: 1, Value: 200}, 104 {ColumnID: 2, Value: "你好,世界"}, 105 }, 106 }, 107 }, 108 }, 109 }, 110 }) 111 require.Nil(t, err) 112 cancel() 113 require.ErrorIs(t, eg.Wait(), context.Canceled) 114 } 115 116 func TestEncodingWorkerRun(t *testing.T) { 117 t.Parallel() 118 119 encodingWorker, msgCh, encodedCh := testEncodingWorker(t) 120 ctx, cancel := context.WithCancel(context.Background()) 121 eg, egCtx := errgroup.WithContext(ctx) 122 outputChs := []*chann.DrainableChann[eventFragment]{chann.NewAutoDrainChann[eventFragment]()} 123 defragmenter := newDefragmenter(encodedCh, outputChs) 124 eg.Go(func() error { 125 return defragmenter.run(egCtx) 126 }) 127 128 table := model.TableName{ 129 Schema: "test", 130 Table: "table1", 131 TableID: 100, 132 } 133 tidbTableInfo := &timodel.TableInfo{ 134 ID: 100, 135 Name: timodel.NewCIStr("table1"), 136 Columns: []*timodel.ColumnInfo{ 137 {ID: 1, Name: timodel.NewCIStr("c1"), FieldType: *types.NewFieldType(mysql.TypeLong)}, 138 {ID: 2, Name: timodel.NewCIStr("c2"), FieldType: *types.NewFieldType(mysql.TypeVarchar)}, 139 }, 140 } 141 tableInfo := model.WrapTableInfo(100, "test", 33, tidbTableInfo) 142 event := &model.SingleTableTxn{ 143 TableInfo: tableInfo, 144 Rows: []*model.RowChangedEvent{ 145 { 146 PhysicalTableID: 100, 147 TableInfo: tableInfo, 148 Columns: []*model.ColumnData{ 149 {ColumnID: 1, Value: 100}, 150 {ColumnID: 2, Value: "hello world"}, 151 }, 152 }, 153 }, 154 } 155 156 for i := 0; i < 3; i++ { 157 frag := eventFragment{ 158 versionedTable: cloudstorage.VersionedTableName{ 159 TableNameWithPhysicTableID: table, 160 }, 161 seqNumber: uint64(i + 1), 162 event: &dmlsink.TxnCallbackableEvent{ 163 Event: event, 164 }, 165 } 166 msgCh <- frag 167 } 168 169 var wg sync.WaitGroup 170 wg.Add(1) 171 go func() { 172 defer wg.Done() 173 _ = encodingWorker.run(ctx) 174 }() 175 176 cancel() 177 encodingWorker.close() 178 wg.Wait() 179 }