github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/canal/canal_entry_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package canal
    15  
    16  import (
    17  	"testing"
    18  
    19  	"github.com/golang/protobuf/proto"
    20  	"github.com/pingcap/tiflow/cdc/entry"
    21  	"github.com/pingcap/tiflow/pkg/config"
    22  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    23  	"github.com/pingcap/tiflow/pkg/sink/codec/internal"
    24  	canal "github.com/pingcap/tiflow/proto/canal"
    25  	"github.com/stretchr/testify/require"
    26  	"golang.org/x/text/encoding/charmap"
    27  )
    28  
    29  func TestInsert(t *testing.T) {
    30  	helper := entry.NewSchemaTestHelper(t)
    31  	defer helper.Close()
    32  
    33  	sql := `create table test.t(
    34  		id int primary key,
    35  		name varchar(32),
    36  		tiny tinyint,
    37  		comment text,
    38  		bb blob)`
    39  	_ = helper.DDL2Event(sql)
    40  
    41  	event := helper.DML2Event(`insert into test.t values(1, "Bob", 127, "测试", "测试blob")`, "test", "t")
    42  
    43  	codecConfig := common.NewConfig(config.ProtocolCanalJSON)
    44  	builder := newCanalEntryBuilder(codecConfig)
    45  	entry, err := builder.fromRowEvent(event, false)
    46  	require.NoError(t, err)
    47  	require.Equal(t, canal.EntryType_ROWDATA, entry.GetEntryType())
    48  	header := entry.GetHeader()
    49  	require.Equal(t, int64(event.CommitTs>>18), header.GetExecuteTime())
    50  	require.Equal(t, canal.Type_MYSQL, header.GetSourceType())
    51  	require.Equal(t, event.TableInfo.GetSchemaName(), header.GetSchemaName())
    52  	require.Equal(t, event.TableInfo.GetTableName(), header.GetTableName())
    53  	require.Equal(t, canal.EventType_INSERT, header.GetEventType())
    54  	store := entry.GetStoreValue()
    55  	require.NotNil(t, store)
    56  	rc := &canal.RowChange{}
    57  	err = proto.Unmarshal(store, rc)
    58  	require.NoError(t, err)
    59  	require.False(t, rc.GetIsDdl())
    60  	rowDatas := rc.GetRowDatas()
    61  	require.Equal(t, 1, len(rowDatas))
    62  
    63  	columns := rowDatas[0].AfterColumns
    64  	require.Equal(t, len(event.Columns), len(columns))
    65  	for _, col := range columns {
    66  		require.True(t, col.GetUpdated())
    67  		switch col.GetName() {
    68  		case "id":
    69  			require.Equal(t, int32(internal.JavaSQLTypeINTEGER), col.GetSqlType())
    70  			require.True(t, col.GetIsKey())
    71  			require.False(t, col.GetIsNull())
    72  			require.Equal(t, "1", col.GetValue())
    73  			require.Equal(t, "int", col.GetMysqlType())
    74  		case "name":
    75  			require.Equal(t, int32(internal.JavaSQLTypeVARCHAR), col.GetSqlType())
    76  			require.False(t, col.GetIsKey())
    77  			require.False(t, col.GetIsNull())
    78  			require.Equal(t, "Bob", col.GetValue())
    79  			require.Equal(t, "varchar", col.GetMysqlType())
    80  		case "tiny":
    81  			require.Equal(t, int32(internal.JavaSQLTypeTINYINT), col.GetSqlType())
    82  			require.False(t, col.GetIsKey())
    83  			require.False(t, col.GetIsNull())
    84  			require.Equal(t, "127", col.GetValue())
    85  		case "comment":
    86  			require.Equal(t, int32(internal.JavaSQLTypeCLOB), col.GetSqlType())
    87  			require.False(t, col.GetIsKey())
    88  			require.False(t, col.GetIsNull())
    89  			require.NoError(t, err)
    90  			require.Equal(t, "测试", col.GetValue())
    91  			require.Equal(t, "text", col.GetMysqlType())
    92  		case "bb":
    93  			require.Equal(t, int32(internal.JavaSQLTypeBLOB), col.GetSqlType())
    94  			require.False(t, col.GetIsKey())
    95  			require.False(t, col.GetIsNull())
    96  			s, err := charmap.ISO8859_1.NewEncoder().String(col.GetValue())
    97  			require.NoError(t, err)
    98  			require.Equal(t, "测试blob", s)
    99  			require.Equal(t, "blob", col.GetMysqlType())
   100  		}
   101  	}
   102  }
   103  
   104  func TestUpdate(t *testing.T) {
   105  	helper := entry.NewSchemaTestHelper(t)
   106  	defer helper.Close()
   107  
   108  	sql := `create table test.t(id int primary key, name varchar(32))`
   109  	_ = helper.DDL2Event(sql)
   110  
   111  	oldEvent := helper.DML2Event(`insert into test.t values (1, "Nancy")`, "test", "t")
   112  	event := helper.DML2Event(`insert into test.t values (2, "Bob")`, "test", "t")
   113  	event.PreColumns = oldEvent.Columns
   114  
   115  	codecConfig := common.NewConfig(config.ProtocolCanalJSON)
   116  	builder := newCanalEntryBuilder(codecConfig)
   117  	entry, err := builder.fromRowEvent(event, false)
   118  	require.NoError(t, err)
   119  	require.Equal(t, canal.EntryType_ROWDATA, entry.GetEntryType())
   120  
   121  	header := entry.GetHeader()
   122  	require.Equal(t, int64(event.CommitTs>>18), header.GetExecuteTime())
   123  	require.Equal(t, canal.Type_MYSQL, header.GetSourceType())
   124  	require.Equal(t, event.TableInfo.GetSchemaName(), header.GetSchemaName())
   125  	require.Equal(t, event.TableInfo.GetTableName(), header.GetTableName())
   126  	require.Equal(t, canal.EventType_UPDATE, header.GetEventType())
   127  	store := entry.GetStoreValue()
   128  	require.NotNil(t, store)
   129  	rc := &canal.RowChange{}
   130  	err = proto.Unmarshal(store, rc)
   131  	require.NoError(t, err)
   132  	require.False(t, rc.GetIsDdl())
   133  	rowDatas := rc.GetRowDatas()
   134  	require.Equal(t, 1, len(rowDatas))
   135  
   136  	beforeColumns := rowDatas[0].BeforeColumns
   137  	require.Equal(t, len(event.PreColumns), len(beforeColumns))
   138  	for _, col := range beforeColumns {
   139  		require.True(t, col.GetUpdated())
   140  		switch col.GetName() {
   141  		case "id":
   142  			require.Equal(t, int32(internal.JavaSQLTypeINTEGER), col.GetSqlType())
   143  			require.True(t, col.GetIsKey())
   144  			require.False(t, col.GetIsNull())
   145  			require.Equal(t, "1", col.GetValue())
   146  			require.Equal(t, "int", col.GetMysqlType())
   147  		case "name":
   148  			require.Equal(t, int32(internal.JavaSQLTypeVARCHAR), col.GetSqlType())
   149  			require.False(t, col.GetIsKey())
   150  			require.False(t, col.GetIsNull())
   151  			require.Equal(t, "Nancy", col.GetValue())
   152  			require.Equal(t, "varchar", col.GetMysqlType())
   153  		}
   154  	}
   155  
   156  	afterColumns := rowDatas[0].AfterColumns
   157  	require.Equal(t, len(event.Columns), len(afterColumns))
   158  	for _, col := range afterColumns {
   159  		require.True(t, col.GetUpdated())
   160  		switch col.GetName() {
   161  		case "id":
   162  			require.Equal(t, int32(internal.JavaSQLTypeINTEGER), col.GetSqlType())
   163  			require.True(t, col.GetIsKey())
   164  			require.False(t, col.GetIsNull())
   165  			require.Equal(t, "2", col.GetValue())
   166  			require.Equal(t, "int", col.GetMysqlType())
   167  		case "name":
   168  			require.Equal(t, int32(internal.JavaSQLTypeVARCHAR), col.GetSqlType())
   169  			require.False(t, col.GetIsKey())
   170  			require.False(t, col.GetIsNull())
   171  			require.Equal(t, "Bob", col.GetValue())
   172  			require.Equal(t, "varchar", col.GetMysqlType())
   173  		}
   174  	}
   175  }
   176  
   177  func TestDelete(t *testing.T) {
   178  	helper := entry.NewSchemaTestHelper(t)
   179  	defer helper.Close()
   180  
   181  	sql := `create table test.t(id int primary key)`
   182  	_ = helper.DDL2Event(sql)
   183  
   184  	event := helper.DML2Event(`insert into test.t values(1)`, "test", "t")
   185  	event.PreColumns = event.Columns
   186  	event.Columns = nil
   187  
   188  	codecConfig := common.NewConfig(config.ProtocolCanalJSON)
   189  	builder := newCanalEntryBuilder(codecConfig)
   190  	entry, err := builder.fromRowEvent(event, false)
   191  	require.NoError(t, err)
   192  	require.Equal(t, canal.EntryType_ROWDATA, entry.GetEntryType())
   193  	header := entry.GetHeader()
   194  	require.Equal(t, event.TableInfo.GetSchemaName(), header.GetSchemaName())
   195  	require.Equal(t, event.TableInfo.GetTableName(), header.GetTableName())
   196  	require.Equal(t, canal.EventType_DELETE, header.GetEventType())
   197  	store := entry.GetStoreValue()
   198  	require.NotNil(t, store)
   199  	rc := &canal.RowChange{}
   200  	err = proto.Unmarshal(store, rc)
   201  	require.NoError(t, err)
   202  	require.False(t, rc.GetIsDdl())
   203  	rowDatas := rc.GetRowDatas()
   204  	require.Equal(t, 1, len(rowDatas))
   205  
   206  	columns := rowDatas[0].BeforeColumns
   207  	require.Equal(t, len(event.PreColumns), len(columns))
   208  	for _, col := range columns {
   209  		require.False(t, col.GetUpdated())
   210  		switch col.GetName() {
   211  		case "id":
   212  			require.Equal(t, int32(internal.JavaSQLTypeINTEGER), col.GetSqlType())
   213  			require.True(t, col.GetIsKey())
   214  			require.False(t, col.GetIsNull())
   215  			require.Equal(t, "1", col.GetValue())
   216  			require.Equal(t, "int", col.GetMysqlType())
   217  		}
   218  	}
   219  }
   220  
   221  func TestDDL(t *testing.T) {
   222  	helper := entry.NewSchemaTestHelper(t)
   223  	defer helper.Close()
   224  
   225  	sql := `create table test.person(id int, name varchar(32), tiny tinyint unsigned, comment text, primary key(id))`
   226  	event := helper.DDL2Event(sql)
   227  
   228  	builder := newCanalEntryBuilder(nil)
   229  	entry, err := builder.fromDDLEvent(event)
   230  	require.NoError(t, err)
   231  	require.Equal(t, canal.EntryType_ROWDATA, entry.GetEntryType())
   232  	header := entry.GetHeader()
   233  	require.Equal(t, event.TableInfo.TableName.Schema, header.GetSchemaName())
   234  	require.Equal(t, event.TableInfo.TableName.Table, header.GetTableName())
   235  	require.Equal(t, canal.EventType_CREATE, header.GetEventType())
   236  	store := entry.GetStoreValue()
   237  	require.NotNil(t, store)
   238  	rc := &canal.RowChange{}
   239  	err = proto.Unmarshal(store, rc)
   240  	require.NoError(t, err)
   241  	require.True(t, rc.GetIsDdl())
   242  	require.Equal(t, event.TableInfo.TableName.Schema, rc.GetDdlSchemaName())
   243  }