github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/redo/writer/memory/mem_log_writer_test.go (about)

     1  //  Copyright 2023 PingCAP, Inc.
     2  //
     3  //  Licensed under the Apache License, Version 2.0 (the "License");
     4  //  you may not use this file except in compliance with the License.
     5  //  You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  //  Unless required by applicable law or agreed to in writing, software
    10  //  distributed under the License is distributed on an "AS IS" BASIS,
    11  //  See the License for the specific language governing permissions and
    12  //  limitations under the License.
    13  
    14  package memory
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"testing"
    20  	"time"
    21  
    22  	"github.com/pingcap/log"
    23  	"github.com/pingcap/tiflow/cdc/model"
    24  	"github.com/pingcap/tiflow/cdc/redo/writer"
    25  	"github.com/pingcap/tiflow/pkg/redo"
    26  	"github.com/pingcap/tiflow/pkg/util"
    27  	"github.com/stretchr/testify/require"
    28  )
    29  
    30  func TestWriteDDL(t *testing.T) {
    31  	t.Parallel()
    32  
    33  	rows := []writer.RedoEvent{
    34  		nil,
    35  		&model.RowChangedEvent{
    36  			PhysicalTableID: 11,
    37  			CommitTs:        11,
    38  			TableInfo:       &model.TableInfo{TableName: model.TableName{Schema: "test", Table: "t1"}},
    39  		},
    40  		&model.RowChangedEvent{
    41  			PhysicalTableID: 12,
    42  			CommitTs:        15,
    43  			TableInfo:       &model.TableInfo{TableName: model.TableName{Schema: "test", Table: "t2"}},
    44  		},
    45  		&model.RowChangedEvent{
    46  			PhysicalTableID: 12,
    47  			CommitTs:        8,
    48  			TableInfo:       &model.TableInfo{TableName: model.TableName{Schema: "test", Table: "t2"}},
    49  		},
    50  	}
    51  	testWriteEvents(t, rows)
    52  }
    53  
    54  func TestWriteDML(t *testing.T) {
    55  	t.Parallel()
    56  
    57  	ddls := []writer.RedoEvent{
    58  		nil,
    59  		&model.DDLEvent{CommitTs: 1},
    60  		&model.DDLEvent{CommitTs: 10},
    61  		&model.DDLEvent{CommitTs: 8},
    62  	}
    63  	testWriteEvents(t, ddls)
    64  }
    65  
    66  func testWriteEvents(t *testing.T, events []writer.RedoEvent) {
    67  	ctx, cancel := context.WithCancel(context.Background())
    68  	defer cancel()
    69  
    70  	extStorage, uri, err := util.GetTestExtStorage(ctx, t.TempDir())
    71  	require.NoError(t, err)
    72  	lwcfg := &writer.LogWriterConfig{
    73  		LogType:            redo.RedoDDLLogFileType,
    74  		CaptureID:          "test-capture",
    75  		ChangeFeedID:       model.DefaultChangeFeedID("test-changefeed"),
    76  		URI:                uri,
    77  		UseExternalStorage: true,
    78  		MaxLogSizeInBytes:  10 * redo.Megabyte,
    79  	}
    80  	filename := t.Name()
    81  	lw, err := NewLogWriter(ctx, lwcfg, writer.WithLogFileName(func() string {
    82  		return filename
    83  	}))
    84  	require.NoError(t, err)
    85  
    86  	require.NoError(t, lw.WriteEvents(ctx, events...))
    87  	require.Eventually(t, func() bool {
    88  		if len(lw.encodeWorkers.outputCh) != 0 {
    89  			log.Warn(fmt.Sprintf("eventCh len %d", len(lw.encodeWorkers.outputCh)))
    90  		}
    91  		return len(lw.encodeWorkers.outputCh) == 0
    92  	}, 2*time.Second, 10*time.Millisecond)
    93  
    94  	// test flush
    95  	require.NoError(t, lw.FlushLog(ctx))
    96  	err = extStorage.WalkDir(ctx, nil, func(path string, size int64) error {
    97  		require.Equal(t, filename, path)
    98  		return nil
    99  	})
   100  	require.NoError(t, err)
   101  
   102  	require.ErrorIs(t, lw.Close(), context.Canceled)
   103  
   104  	err = lw.WriteEvents(ctx, events...)
   105  	require.NoError(t, err)
   106  	err = lw.FlushLog(ctx)
   107  	require.NoError(t, err)
   108  }