github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/ddlsink/cloudstorage/cloud_storage_ddl_sink_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package cloudstorage
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"net/url"
    20  	"os"
    21  	"path"
    22  	"sync/atomic"
    23  	"testing"
    24  	"time"
    25  
    26  	timodel "github.com/pingcap/tidb/pkg/parser/model"
    27  	"github.com/pingcap/tidb/pkg/parser/mysql"
    28  	"github.com/pingcap/tidb/pkg/parser/types"
    29  	"github.com/pingcap/tiflow/cdc/model"
    30  	"github.com/pingcap/tiflow/pkg/config"
    31  	"github.com/pingcap/tiflow/pkg/util"
    32  	"github.com/stretchr/testify/require"
    33  )
    34  
    35  func TestWriteDDLEvent(t *testing.T) {
    36  	ctx, cancel := context.WithCancel(context.Background())
    37  	defer cancel()
    38  	parentDir := t.TempDir()
    39  	uri := fmt.Sprintf("file:///%s?protocol=csv", parentDir)
    40  	sinkURI, err := url.Parse(uri)
    41  	require.Nil(t, err)
    42  	replicaConfig := config.GetDefaultReplicaConfig()
    43  	err = replicaConfig.ValidateAndAdjust(sinkURI)
    44  	require.Nil(t, err)
    45  	sink, err := NewDDLSink(ctx, model.DefaultChangeFeedID("test"), sinkURI, replicaConfig)
    46  	require.Nil(t, err)
    47  
    48  	ddlEvent := &model.DDLEvent{
    49  		CommitTs: 100,
    50  		Type:     timodel.ActionAddColumn,
    51  		Query:    "alter table test.table1 add col2 varchar(64)",
    52  		TableInfo: &model.TableInfo{
    53  			Version: 100,
    54  			TableName: model.TableName{
    55  				Schema:  "test",
    56  				Table:   "table1",
    57  				TableID: 20,
    58  			},
    59  			TableInfo: &timodel.TableInfo{
    60  				Columns: []*timodel.ColumnInfo{
    61  					{
    62  						Name:      timodel.NewCIStr("col1"),
    63  						FieldType: *types.NewFieldType(mysql.TypeLong),
    64  					},
    65  					{
    66  						Name:      timodel.NewCIStr("col2"),
    67  						FieldType: *types.NewFieldType(mysql.TypeVarchar),
    68  					},
    69  				},
    70  			},
    71  		},
    72  	}
    73  	tableDir := path.Join(parentDir, "test/table1/meta/")
    74  	err = sink.WriteDDLEvent(ctx, ddlEvent)
    75  	require.Nil(t, err)
    76  
    77  	tableSchema, err := os.ReadFile(path.Join(tableDir, "schema_100_4192708364.json"))
    78  	require.Nil(t, err)
    79  	require.JSONEq(t, `{
    80  		"Table": "table1",
    81  		"Schema": "test",
    82  		"Version": 1,
    83  		"TableVersion": 100,
    84  		"Query": "alter table test.table1 add col2 varchar(64)",
    85  		"Type": 5,
    86  		"TableColumns": [
    87  			{
    88  				"ColumnName": "col1",
    89  				"ColumnType": "INT",
    90  				"ColumnPrecision": "11"
    91  			},
    92  			{
    93  				"ColumnName": "col2",
    94  				"ColumnType": "VARCHAR",
    95  				"ColumnPrecision": "5"
    96  			}
    97  		],
    98  		"TableColumnsTotal": 2
    99  	}`, string(tableSchema))
   100  }
   101  
   102  func TestWriteCheckpointTs(t *testing.T) {
   103  	ctx, cancel := context.WithCancel(context.Background())
   104  	defer cancel()
   105  	parentDir := t.TempDir()
   106  	uri := fmt.Sprintf("file:///%s?protocol=csv", parentDir)
   107  	sinkURI, err := url.Parse(uri)
   108  	require.Nil(t, err)
   109  	replicaConfig := config.GetDefaultReplicaConfig()
   110  	err = replicaConfig.ValidateAndAdjust(sinkURI)
   111  	require.Nil(t, err)
   112  	sink, err := NewDDLSink(ctx, model.DefaultChangeFeedID("test"), sinkURI, replicaConfig)
   113  	require.Nil(t, err)
   114  	tables := []*model.TableInfo{
   115  		{
   116  			Version: 100,
   117  			TableName: model.TableName{
   118  				Schema:  "test",
   119  				Table:   "table1",
   120  				TableID: 20,
   121  			},
   122  			TableInfo: &timodel.TableInfo{
   123  				Columns: []*timodel.ColumnInfo{
   124  					{
   125  						Name:      timodel.NewCIStr("col1"),
   126  						FieldType: *types.NewFieldType(mysql.TypeLong),
   127  					},
   128  					{
   129  						Name:      timodel.NewCIStr("col2"),
   130  						FieldType: *types.NewFieldType(mysql.TypeVarchar),
   131  					},
   132  				},
   133  			},
   134  		},
   135  	}
   136  
   137  	time.Sleep(3 * time.Second)
   138  	err = sink.WriteCheckpointTs(ctx, 100, tables)
   139  	require.Nil(t, err)
   140  	metadata, err := os.ReadFile(path.Join(parentDir, "metadata"))
   141  	require.Nil(t, err)
   142  	require.JSONEq(t, `{"checkpoint-ts":100}`, string(metadata))
   143  }
   144  
   145  func TestCleanupExpiredFiles(t *testing.T) {
   146  	t.Parallel()
   147  
   148  	ctx, cancel := context.WithCancel(context.Background())
   149  	defer cancel()
   150  	parentDir := t.TempDir()
   151  	uri := fmt.Sprintf("file:///%s?protocol=csv", parentDir)
   152  	sinkURI, err := url.Parse(uri)
   153  	require.Nil(t, err)
   154  	replicaConfig := config.GetDefaultReplicaConfig()
   155  	replicaConfig.Sink.CloudStorageConfig = &config.CloudStorageConfig{
   156  		FileExpirationDays:  util.AddressOf(1),
   157  		FileCleanupCronSpec: util.AddressOf("* * * * * *"),
   158  	}
   159  	err = replicaConfig.ValidateAndAdjust(sinkURI)
   160  	require.Nil(t, err)
   161  
   162  	cnt := atomic.Int64{}
   163  	cleanupJobs := []func(){
   164  		func() {
   165  			cnt.Add(1)
   166  		},
   167  	}
   168  	sink, err := newDDLSink(ctx, model.DefaultChangeFeedID("test"), sinkURI, replicaConfig, cleanupJobs)
   169  	require.Nil(t, err)
   170  
   171  	_ = sink
   172  	time.Sleep(3 * time.Second)
   173  	require.LessOrEqual(t, int64(1), cnt.Load())
   174  }