github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/craft/craft_encoder_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package craft
    15  
    16  import (
    17  	"context"
    18  	"testing"
    19  
    20  	"github.com/pingcap/tiflow/cdc/entry"
    21  	"github.com/pingcap/tiflow/cdc/model"
    22  	"github.com/pingcap/tiflow/pkg/config"
    23  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    24  	"github.com/pingcap/tiflow/pkg/sink/codec/internal"
    25  	"github.com/stretchr/testify/require"
    26  )
    27  
    28  func TestCraftMaxMessageBytes(t *testing.T) {
    29  	cfg := common.NewConfig(config.ProtocolCraft).WithMaxMessageBytes(256)
    30  	encoder := NewBatchEncoderBuilder(cfg).Build()
    31  
    32  	helper := entry.NewSchemaTestHelper(t)
    33  	defer helper.Close()
    34  
    35  	_ = helper.DDL2Event(`create table test.t(a varchar(10) primary key)`)
    36  	testEvent := helper.DML2Event(`insert into test.t values ("aa")`, "test", "t")
    37  
    38  	for i := 0; i < 10000; i++ {
    39  		err := encoder.AppendRowChangedEvent(context.Background(), "", testEvent, nil)
    40  		require.Nil(t, err)
    41  	}
    42  
    43  	messages := encoder.Build()
    44  	for _, msg := range messages {
    45  		require.LessOrEqual(t, msg.Length(), 256)
    46  	}
    47  }
    48  
    49  func TestCraftMaxBatchSize(t *testing.T) {
    50  	cfg := common.NewConfig(config.ProtocolCraft).WithMaxMessageBytes(10485760)
    51  	cfg.MaxBatchSize = 64
    52  	encoder := NewBatchEncoderBuilder(cfg).Build()
    53  
    54  	helper := entry.NewSchemaTestHelper(t)
    55  	defer helper.Close()
    56  
    57  	_ = helper.DDL2Event(`create table test.t(a varchar(10) primary key)`)
    58  	testEvent := helper.DML2Event(`insert into test.t values ("aa")`, "test", "t")
    59  
    60  	for i := 0; i < 10000; i++ {
    61  		err := encoder.AppendRowChangedEvent(context.Background(), "", testEvent, nil)
    62  		require.Nil(t, err)
    63  	}
    64  
    65  	messages := encoder.Build()
    66  	sum := 0
    67  	for _, msg := range messages {
    68  		decoder, err := newBatchDecoder(nil, msg.Value)
    69  		require.Nil(t, err)
    70  		count := 0
    71  		for {
    72  			v, hasNext, err := decoder.HasNext()
    73  			require.Nil(t, err)
    74  			if !hasNext {
    75  				break
    76  			}
    77  
    78  			require.Equal(t, model.MessageTypeRow, v)
    79  			_, err = decoder.NextRowChangedEvent()
    80  			require.NoError(t, err)
    81  			count++
    82  		}
    83  		require.LessOrEqual(t, count, 64)
    84  		sum += count
    85  	}
    86  	require.Equal(t, 10000, sum)
    87  }
    88  
    89  func TestBuildCraftBatchEncoder(t *testing.T) {
    90  	t.Parallel()
    91  	cfg := common.NewConfig(config.ProtocolCraft)
    92  
    93  	builder := &batchEncoderBuilder{config: cfg}
    94  	encoder, ok := builder.Build().(*BatchEncoder)
    95  	require.True(t, ok)
    96  	require.NotNil(t, encoder.config)
    97  }
    98  
    99  func TestDefaultCraftBatchCodec(t *testing.T) {
   100  	cfg := common.NewConfig(config.ProtocolCraft).WithMaxMessageBytes(8192)
   101  	cfg.MaxBatchSize = 64
   102  	builder := NewBatchEncoderBuilder(cfg)
   103  	internal.TestBatchCodec(t, builder, newBatchDecoder)
   104  }
   105  
   106  func TestCraftAppendRowChangedEventWithCallback(t *testing.T) {
   107  	cfg := common.NewConfig(config.ProtocolCraft).WithMaxMessageBytes(10485760)
   108  	cfg.MaxBatchSize = 2
   109  	encoder := NewBatchEncoderBuilder(cfg).Build()
   110  	require.NotNil(t, encoder)
   111  
   112  	count := 0
   113  
   114  	helper := entry.NewSchemaTestHelper(t)
   115  	defer helper.Close()
   116  
   117  	_ = helper.DDL2Event(`create table test.t(a varchar(10) primary key)`)
   118  	row := helper.DML2Event(`insert into test.t values ("aa")`, "test", "t")
   119  
   120  	tests := []struct {
   121  		row      *model.RowChangedEvent
   122  		callback func()
   123  	}{
   124  		{
   125  			row: row,
   126  			callback: func() {
   127  				count += 1
   128  			},
   129  		},
   130  		{
   131  			row: row,
   132  			callback: func() {
   133  				count += 2
   134  			},
   135  		},
   136  		{
   137  			row: row,
   138  			callback: func() {
   139  				count += 3
   140  			},
   141  		},
   142  		{
   143  			row: row,
   144  			callback: func() {
   145  				count += 4
   146  			},
   147  		},
   148  		{
   149  			row: row,
   150  			callback: func() {
   151  				count += 5
   152  			},
   153  		},
   154  	}
   155  
   156  	// Empty build makes sure that the callback build logic not broken.
   157  	msgs := encoder.Build()
   158  	require.Len(t, msgs, 0, "no message should be built and no panic")
   159  
   160  	// Append the events.
   161  	for _, test := range tests {
   162  		err := encoder.AppendRowChangedEvent(context.Background(), "", test.row, test.callback)
   163  		require.Nil(t, err)
   164  	}
   165  	require.Equal(t, 0, count, "nothing should be called")
   166  
   167  	msgs = encoder.Build()
   168  	require.Len(t, msgs, 3, "expected 3 messages")
   169  	msgs[0].Callback()
   170  	require.Equal(t, 3, count, "expected 2 callbacks to be called")
   171  	msgs[1].Callback()
   172  	require.Equal(t, 10, count, "expected 2 callbacks to be called")
   173  	msgs[2].Callback()
   174  	require.Equal(t, 15, count, "expected one callback to be called")
   175  }