github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/debezium/codec_test.go (about)

     1  // Copyright 2024 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package debezium
    15  
    16  import (
    17  	"bytes"
    18  	"testing"
    19  	"time"
    20  
    21  	"github.com/pingcap/tidb/pkg/parser/mysql"
    22  	"github.com/pingcap/tiflow/cdc/model"
    23  	"github.com/pingcap/tiflow/pkg/config"
    24  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    25  	"github.com/stretchr/testify/require"
    26  	"github.com/thanhpk/randstr"
    27  )
    28  
    29  func TestEncodeInsert(t *testing.T) {
    30  	codec := &dbzCodec{
    31  		config:    common.NewConfig(config.ProtocolDebezium),
    32  		clusterID: "test-cluster",
    33  		nowFunc:   func() time.Time { return time.Unix(1701326309, 0) },
    34  	}
    35  	codec.config.DebeziumDisableSchema = true
    36  	codec.config.DebeziumOutputOldValue = false
    37  
    38  	tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{
    39  		Name: "tiny",
    40  		Type: mysql.TypeTiny,
    41  		Flag: model.NullableFlag,
    42  	}}, nil)
    43  	e := &model.RowChangedEvent{
    44  		CommitTs:  1,
    45  		TableInfo: tableInfo,
    46  		Columns: model.Columns2ColumnDatas([]*model.Column{{
    47  			Name:  "tiny",
    48  			Value: int64(1),
    49  		}}, tableInfo),
    50  	}
    51  
    52  	buf := bytes.NewBuffer(nil)
    53  	err := codec.EncodeRowChangedEvent(e, buf)
    54  	require.Nil(t, err)
    55  	require.JSONEq(t, `
    56  	{
    57  		"payload": {
    58  			"before": null,
    59  			"after": {
    60  				"tiny": 1
    61  			},
    62  			"op": "c",
    63  			"source": {
    64  				"cluster_id": "test-cluster",
    65  				"name": "test-cluster",
    66  				"commit_ts": 1,
    67  				"connector": "TiCDC",
    68  				"db": "test",
    69  				"table": "table1",
    70  				"ts_ms": 0,
    71  				"file": "",
    72  				"gtid": null,
    73  				"pos": 0,
    74  				"query": null,
    75  				"row": 0,
    76  				"server_id": 0,
    77  				"snapshot": "false",
    78  				"thread": 0,
    79  				"version": "2.4.0.Final"
    80  			},
    81  			"ts_ms": 1701326309000,
    82  			"transaction": null
    83  		}
    84  	}
    85  	`, buf.String())
    86  
    87  	codec.config.DebeziumDisableSchema = false
    88  	buf.Reset()
    89  	err = codec.EncodeRowChangedEvent(e, buf)
    90  	require.Nil(t, err)
    91  	require.JSONEq(t, `
    92  	{
    93  		"payload": {
    94  			"source": {
    95  				"version": "2.4.0.Final",
    96  				"connector": "TiCDC",
    97  				"name": "test-cluster",
    98  				"ts_ms": 0,
    99  				"snapshot": "false",
   100  				"db": "test",
   101  				"table": "table1",
   102  				"server_id": 0,
   103  				"gtid": null,
   104  				"file": "",
   105  				"pos": 0,
   106  				"row": 0,
   107  				"thread": 0,
   108  				"query": null,
   109  				"commit_ts": 1,
   110  				"cluster_id": "test-cluster"
   111  			},
   112  			"ts_ms": 1701326309000,
   113  			"transaction": null,
   114  			"op": "c",
   115  			"before": null,
   116  			"after": { "tiny": 1 }
   117  		},
   118  		"schema": {
   119  			"type": "struct",
   120  			"optional": false,
   121  			"name": "test-cluster.test.table1.Envelope",
   122  			"version": 1,
   123  			"fields": [
   124  				{
   125  					"type": "struct",
   126  					"optional": true,
   127  					"name": "test-cluster.test.table1.Value",
   128  					"field": "before",
   129  					"fields": [{ "type": "int16", "optional": true, "field": "tiny" }]
   130  				},
   131  				{
   132  					"type": "struct",
   133  					"optional": true,
   134  					"name": "test-cluster.test.table1.Value",
   135  					"field": "after",
   136  					"fields": [{ "type": "int16", "optional": true, "field": "tiny" }]
   137  				},
   138  				{
   139  					"type": "struct",
   140  					"fields": [
   141  						{ "type": "string", "optional": false, "field": "version" },
   142  						{ "type": "string", "optional": false, "field": "connector" },
   143  						{ "type": "string", "optional": false, "field": "name" },
   144  						{ "type": "int64", "optional": false, "field": "ts_ms" },
   145  						{
   146  							"type": "string",
   147  							"optional": true,
   148  							"name": "io.debezium.data.Enum",
   149  							"version": 1,
   150  							"parameters": { "allowed": "true,last,false,incremental" },
   151  							"default": "false",
   152  							"field": "snapshot"
   153  						},
   154  						{ "type": "string", "optional": false, "field": "db" },
   155  						{ "type": "string", "optional": true, "field": "sequence" },
   156  						{ "type": "string", "optional": true, "field": "table" },
   157  						{ "type": "int64", "optional": false, "field": "server_id" },
   158  						{ "type": "string", "optional": true, "field": "gtid" },
   159  						{ "type": "string", "optional": false, "field": "file" },
   160  						{ "type": "int64", "optional": false, "field": "pos" },
   161  						{ "type": "int32", "optional": false, "field": "row" },
   162  						{ "type": "int64", "optional": true, "field": "thread" },
   163  						{ "type": "string", "optional": true, "field": "query" }
   164  					],
   165  					"optional": false,
   166  					"name": "io.debezium.connector.mysql.Source",
   167  					"field": "source"
   168  				},
   169  				{ "type": "string", "optional": false, "field": "op" },
   170  				{ "type": "int64", "optional": true, "field": "ts_ms" },
   171  				{
   172  					"type": "struct",
   173  					"fields": [
   174  						{ "type": "string", "optional": false, "field": "id" },
   175  						{ "type": "int64", "optional": false, "field": "total_order" },
   176  						{
   177  							"type": "int64",
   178  							"optional": false,
   179  							"field": "data_collection_order"
   180  						}
   181  					],
   182  					"optional": true,
   183  					"name": "event.block",
   184  					"version": 1,
   185  					"field": "transaction"
   186  				}
   187  			]
   188  		}
   189  	}
   190  	`, buf.String())
   191  }
   192  
   193  func TestEncodeUpdate(t *testing.T) {
   194  	codec := &dbzCodec{
   195  		config:    common.NewConfig(config.ProtocolDebezium),
   196  		clusterID: "test-cluster",
   197  		nowFunc:   func() time.Time { return time.Unix(1701326309, 0) },
   198  	}
   199  	codec.config.DebeziumDisableSchema = true
   200  
   201  	tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{
   202  		Name: "tiny",
   203  		Type: mysql.TypeTiny,
   204  		Flag: model.NullableFlag,
   205  	}}, nil)
   206  	e := &model.RowChangedEvent{
   207  		CommitTs:  1,
   208  		TableInfo: tableInfo,
   209  		Columns: model.Columns2ColumnDatas([]*model.Column{{
   210  			Name:  "tiny",
   211  			Value: int64(1),
   212  		}}, tableInfo),
   213  		PreColumns: model.Columns2ColumnDatas([]*model.Column{{
   214  			Name:  "tiny",
   215  			Value: int64(2),
   216  		}}, tableInfo),
   217  	}
   218  
   219  	buf := bytes.NewBuffer(nil)
   220  	err := codec.EncodeRowChangedEvent(e, buf)
   221  	require.Nil(t, err)
   222  	require.JSONEq(t, `
   223  	{
   224  		"payload": {
   225  			"before": {
   226  				"tiny": 2
   227  			},
   228  			"after": {
   229  				"tiny": 1
   230  			},
   231  			"op": "u",
   232  			"source": {
   233  				"cluster_id": "test-cluster",
   234  				"name": "test-cluster",
   235  				"commit_ts": 1,
   236  				"connector": "TiCDC",
   237  				"db": "test",
   238  				"table": "table1",
   239  				"ts_ms": 0,
   240  				"file": "",
   241  				"gtid": null,
   242  				"pos": 0,
   243  				"query": null,
   244  				"row": 0,
   245  				"server_id": 0,
   246  				"snapshot": "false",
   247  				"thread": 0,
   248  				"version": "2.4.0.Final"
   249  			},
   250  			"ts_ms": 1701326309000,
   251  			"transaction": null
   252  		}
   253  	}
   254  	`, buf.String())
   255  
   256  	codec.config.DebeziumDisableSchema = false
   257  	buf.Reset()
   258  	err = codec.EncodeRowChangedEvent(e, buf)
   259  	require.Nil(t, err)
   260  	require.JSONEq(t, `
   261  	{
   262  		"payload": {
   263  			"source": {
   264  				"version": "2.4.0.Final",
   265  				"connector": "TiCDC",
   266  				"name": "test-cluster",
   267  				"ts_ms": 0,
   268  				"snapshot": "false",
   269  				"db": "test",
   270  				"table": "table1",
   271  				"server_id": 0,
   272  				"gtid": null,
   273  				"file": "",
   274  				"pos": 0,
   275  				"row": 0,
   276  				"thread": 0,
   277  				"query": null,
   278  				"commit_ts": 1,
   279  				"cluster_id": "test-cluster"
   280  			},
   281  			"ts_ms": 1701326309000,
   282  			"transaction": null,
   283  			"op": "u",
   284  			"before": { "tiny": 2 },
   285  			"after": { "tiny": 1 }
   286  		},
   287  		"schema": {
   288  			"type": "struct",
   289  			"optional": false,
   290  			"name": "test-cluster.test.table1.Envelope",
   291  			"version": 1,
   292  			"fields": [
   293  				{
   294  					"type": "struct",
   295  					"optional": true,
   296  					"name": "test-cluster.test.table1.Value",
   297  					"field": "before",
   298  					"fields": [{ "type": "int16", "optional": true, "field": "tiny" }]
   299  				},
   300  				{
   301  					"type": "struct",
   302  					"optional": true,
   303  					"name": "test-cluster.test.table1.Value",
   304  					"field": "after",
   305  					"fields": [{ "type": "int16", "optional": true, "field": "tiny" }]
   306  				},
   307  				{
   308  					"type": "struct",
   309  					"fields": [
   310  						{ "type": "string", "optional": false, "field": "version" },
   311  						{ "type": "string", "optional": false, "field": "connector" },
   312  						{ "type": "string", "optional": false, "field": "name" },
   313  						{ "type": "int64", "optional": false, "field": "ts_ms" },
   314  						{
   315  							"type": "string",
   316  							"optional": true,
   317  							"name": "io.debezium.data.Enum",
   318  							"version": 1,
   319  							"parameters": { "allowed": "true,last,false,incremental" },
   320  							"default": "false",
   321  							"field": "snapshot"
   322  						},
   323  						{ "type": "string", "optional": false, "field": "db" },
   324  						{ "type": "string", "optional": true, "field": "sequence" },
   325  						{ "type": "string", "optional": true, "field": "table" },
   326  						{ "type": "int64", "optional": false, "field": "server_id" },
   327  						{ "type": "string", "optional": true, "field": "gtid" },
   328  						{ "type": "string", "optional": false, "field": "file" },
   329  						{ "type": "int64", "optional": false, "field": "pos" },
   330  						{ "type": "int32", "optional": false, "field": "row" },
   331  						{ "type": "int64", "optional": true, "field": "thread" },
   332  						{ "type": "string", "optional": true, "field": "query" }
   333  					],
   334  					"optional": false,
   335  					"name": "io.debezium.connector.mysql.Source",
   336  					"field": "source"
   337  				},
   338  				{ "type": "string", "optional": false, "field": "op" },
   339  				{ "type": "int64", "optional": true, "field": "ts_ms" },
   340  				{
   341  					"type": "struct",
   342  					"fields": [
   343  						{ "type": "string", "optional": false, "field": "id" },
   344  						{ "type": "int64", "optional": false, "field": "total_order" },
   345  						{
   346  							"type": "int64",
   347  							"optional": false,
   348  							"field": "data_collection_order"
   349  						}
   350  					],
   351  					"optional": true,
   352  					"name": "event.block",
   353  					"version": 1,
   354  					"field": "transaction"
   355  				}
   356  			]
   357  		}
   358  	}
   359  	`, buf.String())
   360  
   361  	codec.config.DebeziumOutputOldValue = false
   362  	codec.config.DebeziumDisableSchema = true
   363  	buf.Reset()
   364  	err = codec.EncodeRowChangedEvent(e, buf)
   365  	require.Nil(t, err)
   366  	require.JSONEq(t, `
   367  	{
   368  		"payload": {
   369  			"source": {
   370  				"version": "2.4.0.Final",
   371  				"connector": "TiCDC",
   372  				"name": "test-cluster",
   373  				"ts_ms": 0,
   374  				"snapshot": "false",
   375  				"db": "test",
   376  				"table": "table1",
   377  				"server_id": 0,
   378  				"gtid": null,
   379  				"file": "",
   380  				"pos": 0,
   381  				"row": 0,
   382  				"thread": 0,
   383  				"query": null,
   384  				"commit_ts": 1,
   385  				"cluster_id": "test-cluster"
   386  			},
   387  			"ts_ms": 1701326309000,
   388  			"transaction": null,
   389  			"op": "u",
   390  			"after": { "tiny": 1 }
   391  		}
   392  	}
   393  	`, buf.String())
   394  }
   395  
   396  func TestEncodeDelete(t *testing.T) {
   397  	codec := &dbzCodec{
   398  		config:    common.NewConfig(config.ProtocolDebezium),
   399  		clusterID: "test-cluster",
   400  		nowFunc:   func() time.Time { return time.Unix(1701326309, 0) },
   401  	}
   402  	codec.config.DebeziumOutputOldValue = false
   403  	codec.config.DebeziumDisableSchema = true
   404  
   405  	tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{
   406  		Name: "tiny",
   407  		Type: mysql.TypeTiny,
   408  		Flag: model.NullableFlag,
   409  	}}, nil)
   410  	e := &model.RowChangedEvent{
   411  		CommitTs:  1,
   412  		TableInfo: tableInfo,
   413  		PreColumns: model.Columns2ColumnDatas([]*model.Column{{
   414  			Name:  "tiny",
   415  			Value: int64(2),
   416  		}}, tableInfo),
   417  	}
   418  
   419  	buf := bytes.NewBuffer(nil)
   420  	err := codec.EncodeRowChangedEvent(e, buf)
   421  	require.Nil(t, err)
   422  	require.JSONEq(t, `
   423  	{
   424  		"payload": {
   425  			"before": {
   426  				"tiny": 2
   427  			},
   428  			"after": null,
   429  			"op": "d",
   430  			"source": {
   431  				"cluster_id": "test-cluster",
   432  				"name": "test-cluster",
   433  				"commit_ts": 1,
   434  				"connector": "TiCDC",
   435  				"db": "test",
   436  				"table": "table1",
   437  				"ts_ms": 0,
   438  				"file": "",
   439  				"gtid": null,
   440  				"pos": 0,
   441  				"query": null,
   442  				"row": 0,
   443  				"server_id": 0,
   444  				"snapshot": "false",
   445  				"thread": 0,
   446  				"version": "2.4.0.Final"
   447  			},
   448  			"ts_ms": 1701326309000,
   449  			"transaction": null
   450  		}
   451  	}
   452  	`, buf.String())
   453  
   454  	codec.config.DebeziumDisableSchema = false
   455  	buf.Reset()
   456  	err = codec.EncodeRowChangedEvent(e, buf)
   457  	require.Nil(t, err)
   458  	require.JSONEq(t, `
   459  	{
   460  		"payload": {
   461  			"source": {
   462  				"version": "2.4.0.Final",
   463  				"connector": "TiCDC",
   464  				"name": "test-cluster",
   465  				"ts_ms": 0,
   466  				"snapshot": "false",
   467  				"db": "test",
   468  				"table": "table1",
   469  				"server_id": 0,
   470  				"gtid": null,
   471  				"file": "",
   472  				"pos": 0,
   473  				"row": 0,
   474  				"thread": 0,
   475  				"query": null,
   476  				"commit_ts": 1,
   477  				"cluster_id": "test-cluster"
   478  			},
   479  			"ts_ms": 1701326309000,
   480  			"transaction": null,
   481  			"op": "d",
   482  			"after": null,
   483  			"before": { "tiny": 2 }
   484  		},
   485  		"schema": {
   486  			"type": "struct",
   487  			"optional": false,
   488  			"name": "test-cluster.test.table1.Envelope",
   489  			"version": 1,
   490  			"fields": [
   491  				{
   492  					"type": "struct",
   493  					"optional": true,
   494  					"name": "test-cluster.test.table1.Value",
   495  					"field": "before",
   496  					"fields": [{ "type": "int16", "optional": true, "field": "tiny" }]
   497  				},
   498  				{
   499  					"type": "struct",
   500  					"optional": true,
   501  					"name": "test-cluster.test.table1.Value",
   502  					"field": "after",
   503  					"fields": [{ "type": "int16", "optional": true, "field": "tiny" }]
   504  				},
   505  				{
   506  					"type": "struct",
   507  					"fields": [
   508  						{ "type": "string", "optional": false, "field": "version" },
   509  						{ "type": "string", "optional": false, "field": "connector" },
   510  						{ "type": "string", "optional": false, "field": "name" },
   511  						{ "type": "int64", "optional": false, "field": "ts_ms" },
   512  						{
   513  							"type": "string",
   514  							"optional": true,
   515  							"name": "io.debezium.data.Enum",
   516  							"version": 1,
   517  							"parameters": { "allowed": "true,last,false,incremental" },
   518  							"default": "false",
   519  							"field": "snapshot"
   520  						},
   521  						{ "type": "string", "optional": false, "field": "db" },
   522  						{ "type": "string", "optional": true, "field": "sequence" },
   523  						{ "type": "string", "optional": true, "field": "table" },
   524  						{ "type": "int64", "optional": false, "field": "server_id" },
   525  						{ "type": "string", "optional": true, "field": "gtid" },
   526  						{ "type": "string", "optional": false, "field": "file" },
   527  						{ "type": "int64", "optional": false, "field": "pos" },
   528  						{ "type": "int32", "optional": false, "field": "row" },
   529  						{ "type": "int64", "optional": true, "field": "thread" },
   530  						{ "type": "string", "optional": true, "field": "query" }
   531  					],
   532  					"optional": false,
   533  					"name": "io.debezium.connector.mysql.Source",
   534  					"field": "source"
   535  				},
   536  				{ "type": "string", "optional": false, "field": "op" },
   537  				{ "type": "int64", "optional": true, "field": "ts_ms" },
   538  				{
   539  					"type": "struct",
   540  					"fields": [
   541  						{ "type": "string", "optional": false, "field": "id" },
   542  						{ "type": "int64", "optional": false, "field": "total_order" },
   543  						{
   544  							"type": "int64",
   545  							"optional": false,
   546  							"field": "data_collection_order"
   547  						}
   548  					],
   549  					"optional": true,
   550  					"name": "event.block",
   551  					"version": 1,
   552  					"field": "transaction"
   553  				}
   554  			]
   555  		}
   556  	}
   557  	`, buf.String())
   558  }
   559  
   560  func BenchmarkEncodeOneTinyColumn(b *testing.B) {
   561  	codec := &dbzCodec{
   562  		config:    common.NewConfig(config.ProtocolDebezium),
   563  		clusterID: "test-cluster",
   564  		nowFunc:   func() time.Time { return time.Unix(1701326309, 0) },
   565  	}
   566  	codec.config.DebeziumDisableSchema = true
   567  
   568  	tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{
   569  		Name: "tiny",
   570  		Type: mysql.TypeTiny,
   571  	}}, nil)
   572  	e := &model.RowChangedEvent{
   573  		CommitTs:  1,
   574  		TableInfo: tableInfo,
   575  		Columns: model.Columns2ColumnDatas([]*model.Column{{
   576  			Name:  "tiny",
   577  			Value: int64(10),
   578  		}}, tableInfo),
   579  	}
   580  
   581  	buf := bytes.NewBuffer(nil)
   582  
   583  	b.ResetTimer()
   584  	for n := 0; n < b.N; n++ {
   585  		buf.Reset()
   586  		codec.EncodeRowChangedEvent(e, buf)
   587  	}
   588  }
   589  
   590  func BenchmarkEncodeLargeText(b *testing.B) {
   591  	codec := &dbzCodec{
   592  		config:    common.NewConfig(config.ProtocolDebezium),
   593  		clusterID: "test-cluster",
   594  		nowFunc:   func() time.Time { return time.Unix(1701326309, 0) },
   595  	}
   596  	codec.config.DebeziumDisableSchema = true
   597  
   598  	tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{
   599  		Name: "str",
   600  		Type: mysql.TypeVarchar,
   601  	}}, nil)
   602  	e := &model.RowChangedEvent{
   603  		CommitTs:  1,
   604  		TableInfo: tableInfo,
   605  		Columns: model.Columns2ColumnDatas([]*model.Column{{
   606  			Name:  "str",
   607  			Value: []byte(randstr.String(1024)),
   608  		}}, tableInfo),
   609  	}
   610  
   611  	buf := bytes.NewBuffer(nil)
   612  
   613  	b.ResetTimer()
   614  	for n := 0; n < b.N; n++ {
   615  		buf.Reset()
   616  		codec.EncodeRowChangedEvent(e, buf)
   617  	}
   618  }
   619  
   620  func BenchmarkEncodeLargeBinary(b *testing.B) {
   621  	codec := &dbzCodec{
   622  		config:    common.NewConfig(config.ProtocolDebezium),
   623  		clusterID: "test-cluster",
   624  		nowFunc:   func() time.Time { return time.Unix(1701326309, 0) },
   625  	}
   626  	codec.config.DebeziumDisableSchema = true
   627  
   628  	tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{
   629  		Name: "bin",
   630  		Type: mysql.TypeVarchar,
   631  		Flag: model.BinaryFlag,
   632  	}}, nil)
   633  	e := &model.RowChangedEvent{
   634  		CommitTs:  1,
   635  		TableInfo: tableInfo,
   636  		Columns: model.Columns2ColumnDatas([]*model.Column{{
   637  			Name:  "bin",
   638  			Value: []byte(randstr.String(1024)),
   639  		}}, tableInfo),
   640  	}
   641  
   642  	buf := bytes.NewBuffer(nil)
   643  
   644  	b.ResetTimer()
   645  	for n := 0; n < b.N; n++ {
   646  		buf.Reset()
   647  		codec.EncodeRowChangedEvent(e, buf)
   648  	}
   649  }