github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/model/changefeed_test.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package model
    15  
    16  import (
    17  	"fmt"
    18  	"math"
    19  	"strings"
    20  	"testing"
    21  	"time"
    22  
    23  	"github.com/pingcap/tiflow/pkg/config"
    24  	"github.com/pingcap/tiflow/pkg/errors"
    25  	"github.com/pingcap/tiflow/pkg/util"
    26  	"github.com/stretchr/testify/require"
    27  	"github.com/tikv/client-go/v2/oracle"
    28  )
    29  
    30  func TestRmUnusedField(t *testing.T) {
    31  	t.Parallel()
    32  	const (
    33  		defaultRegistry string = "default-schema-registry"
    34  		defaultProtocol string = "default-protocol"
    35  	)
    36  
    37  	// 1. mysql downstream
    38  	{
    39  		mysqlCf := &ChangeFeedInfo{
    40  			SinkURI: "mysql://",
    41  			Config: &config.ReplicaConfig{
    42  				Sink: &config.SinkConfig{
    43  					SchemaRegistry: util.AddressOf(defaultRegistry),
    44  					Protocol:       util.AddressOf(defaultProtocol),
    45  					CSVConfig: &config.CSVConfig{
    46  						Quote:      string(config.DoubleQuoteChar),
    47  						Delimiter:  config.Comma,
    48  						NullString: config.NULL,
    49  					},
    50  				},
    51  			},
    52  		}
    53  
    54  		mysqlCf.VerifyAndComplete()
    55  		require.True(t, mysqlCf.Config.Sink.SchemaRegistry == nil)
    56  		require.True(t, mysqlCf.Config.Sink.Protocol == nil)
    57  		require.Nil(t, mysqlCf.Config.Sink.CSVConfig)
    58  	}
    59  
    60  	// 2. storage downstream
    61  	{
    62  		strCf := &ChangeFeedInfo{
    63  			SinkURI: "s3://",
    64  			Config: &config.ReplicaConfig{
    65  				Sink: &config.SinkConfig{
    66  					SchemaRegistry: util.AddressOf(defaultRegistry),
    67  					Protocol:       util.AddressOf(defaultProtocol),
    68  					CSVConfig: &config.CSVConfig{
    69  						Quote:      string(config.DoubleQuoteChar),
    70  						Delimiter:  config.Comma,
    71  						NullString: config.NULL,
    72  					},
    73  				},
    74  			},
    75  		}
    76  		strCf.VerifyAndComplete()
    77  		require.True(t, strCf.Config.Sink.SchemaRegistry == nil)
    78  		require.NotNil(t, strCf.Config.Sink.CSVConfig)
    79  	}
    80  
    81  	// 3. kafka downstream using avro
    82  	{
    83  		kaCf := &ChangeFeedInfo{
    84  			SinkURI: "kafka://",
    85  			Config: &config.ReplicaConfig{
    86  				Sink: &config.SinkConfig{
    87  					Protocol:       util.AddressOf(config.ProtocolAvro.String()),
    88  					SchemaRegistry: util.AddressOf(defaultRegistry),
    89  					CSVConfig: &config.CSVConfig{
    90  						Quote:      string(config.DoubleQuoteChar),
    91  						Delimiter:  config.Comma,
    92  						NullString: config.NULL,
    93  					},
    94  				},
    95  			},
    96  		}
    97  		kaCf.VerifyAndComplete()
    98  		require.Equal(t, defaultRegistry, util.GetOrZero(kaCf.Config.Sink.SchemaRegistry))
    99  		require.Equal(t, config.ProtocolAvro.String(), util.GetOrZero(kaCf.Config.Sink.Protocol))
   100  		require.Nil(t, kaCf.Config.Sink.CSVConfig)
   101  	}
   102  
   103  	// 4. kafka downstream using canal-json
   104  	{
   105  		kcCf := &ChangeFeedInfo{
   106  			SinkURI: "kafka://",
   107  			Config: &config.ReplicaConfig{
   108  				Sink: &config.SinkConfig{
   109  					Protocol:       util.AddressOf(config.ProtocolCanal.String()),
   110  					SchemaRegistry: util.AddressOf(defaultRegistry),
   111  					CSVConfig: &config.CSVConfig{
   112  						Quote:      string(config.DoubleQuoteChar),
   113  						Delimiter:  config.Comma,
   114  						NullString: config.NULL,
   115  					},
   116  				},
   117  			},
   118  		}
   119  		kcCf.VerifyAndComplete()
   120  		require.True(t, kcCf.Config.Sink.SchemaRegistry == nil)
   121  		require.Equal(
   122  			t,
   123  			config.ProtocolCanal.String(),
   124  			util.GetOrZero(kcCf.Config.Sink.Protocol),
   125  		)
   126  		require.Nil(t, kcCf.Config.Sink.CSVConfig)
   127  	}
   128  }
   129  
   130  func TestFillV1(t *testing.T) {
   131  	t.Parallel()
   132  
   133  	v1Config := `
   134  {
   135      "sink-uri":"blackhole://",
   136      "start-ts":417136892416622595,
   137      "target-ts":0,
   138      "admin-job-type":0,
   139      "sort-engine":"memory",
   140      "sort-dir":".",
   141      "config":{
   142          "case-sensitive":true,
   143          "filter":{
   144              "ignore-txn-start-ts":[
   145                  1,
   146                  2
   147              ]
   148          },
   149          "mounter":{
   150              "worker-num":64
   151          },
   152          "sink":{
   153              "dispatch-rules":[
   154                  {
   155                      "db-name":"test",
   156                      "tbl-name":"tbl3",
   157                      "rule":"ts"
   158                  },
   159                  {
   160                      "db-name":"test",
   161                      "tbl-name":"tbl4",
   162                      "rule":"rowid"
   163                  }
   164              ]
   165          }
   166      }
   167  }
   168  `
   169  	cfg := &ChangeFeedInfo{}
   170  	err := cfg.Unmarshal([]byte(v1Config))
   171  	require.Nil(t, err)
   172  	require.Equal(t, &ChangeFeedInfo{
   173  		SinkURI: "blackhole://",
   174  		StartTs: 417136892416622595,
   175  		Engine:  "memory",
   176  		SortDir: ".",
   177  		Config: &config.ReplicaConfig{
   178  			CaseSensitive: true,
   179  			Filter: &config.FilterConfig{
   180  				IgnoreTxnStartTs: []uint64{1, 2},
   181  			},
   182  			Mounter: &config.MounterConfig{
   183  				WorkerNum: 64,
   184  			},
   185  			Sink: &config.SinkConfig{
   186  				DispatchRules: []*config.DispatchRule{
   187  					{Matcher: []string{"test.tbl3"}, DispatcherRule: "ts"},
   188  					{Matcher: []string{"test.tbl4"}, DispatcherRule: "rowid"},
   189  				},
   190  			},
   191  		},
   192  	}, cfg)
   193  }
   194  
   195  func TestVerifyAndComplete(t *testing.T) {
   196  	t.Parallel()
   197  
   198  	info := &ChangeFeedInfo{
   199  		SinkURI: "mysql://",
   200  		StartTs: 417257993615179777,
   201  		Config: &config.ReplicaConfig{
   202  			MemoryQuota:           1073741824,
   203  			CaseSensitive:         false,
   204  			CheckGCSafePoint:      true,
   205  			EnableSyncPoint:       util.AddressOf(false),
   206  			EnableTableMonitor:    util.AddressOf(false),
   207  			SyncPointInterval:     util.AddressOf(time.Minute * 10),
   208  			SyncPointRetention:    util.AddressOf(time.Hour * 24),
   209  			BDRMode:               util.AddressOf(false),
   210  			IgnoreIneligibleTable: false,
   211  		},
   212  	}
   213  
   214  	info.VerifyAndComplete()
   215  	require.Equal(t, SortUnified, info.Engine)
   216  
   217  	marshalConfig1, err := info.Config.Marshal()
   218  	require.Nil(t, err)
   219  	defaultConfig := config.GetDefaultReplicaConfig()
   220  	info2 := &ChangeFeedInfo{
   221  		SinkURI: "mysql://",
   222  		Config:  defaultConfig,
   223  	}
   224  	info2.RmUnusedFields()
   225  	marshalConfig2, err := defaultConfig.Marshal()
   226  	require.Nil(t, err)
   227  	require.Equal(t, marshalConfig2, marshalConfig1)
   228  }
   229  
   230  func TestFixStateIncompatible(t *testing.T) {
   231  	t.Parallel()
   232  
   233  	// Test to fix incompatible states.
   234  	testCases := []struct {
   235  		info          *ChangeFeedInfo
   236  		expectedState FeedState
   237  	}{
   238  		{
   239  			info: &ChangeFeedInfo{
   240  				AdminJobType:   AdminStop,
   241  				State:          StateNormal,
   242  				Error:          nil,
   243  				CreatorVersion: "",
   244  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   245  				Config: &config.ReplicaConfig{
   246  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   247  				},
   248  			},
   249  			expectedState: StateStopped,
   250  		},
   251  		{
   252  			info: &ChangeFeedInfo{
   253  				AdminJobType:   AdminStop,
   254  				State:          StateNormal,
   255  				Error:          nil,
   256  				CreatorVersion: "4.0.14",
   257  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   258  				Config: &config.ReplicaConfig{
   259  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   260  				},
   261  			},
   262  			expectedState: StateStopped,
   263  		},
   264  		{
   265  			info: &ChangeFeedInfo{
   266  				AdminJobType:   AdminStop,
   267  				State:          StateNormal,
   268  				Error:          nil,
   269  				CreatorVersion: "5.0.5",
   270  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   271  				Config: &config.ReplicaConfig{
   272  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   273  				},
   274  			},
   275  			expectedState: StateStopped,
   276  		},
   277  	}
   278  
   279  	for _, tc := range testCases {
   280  		tc.info.FixIncompatible()
   281  		require.Equal(t, tc.expectedState, tc.info.State)
   282  	}
   283  }
   284  
   285  func TestFixSinkProtocolIncompatible(t *testing.T) {
   286  	t.Parallel()
   287  
   288  	emptyProtocolStr := ""
   289  	// Test to fix incompatible protocols.
   290  	configTestCases := []struct {
   291  		info                *ChangeFeedInfo
   292  		expectedProtocol    config.Protocol
   293  		expectedProtocolStr *string
   294  	}{
   295  		{
   296  			info: &ChangeFeedInfo{
   297  				AdminJobType:   AdminStop,
   298  				State:          StateStopped,
   299  				Error:          nil,
   300  				CreatorVersion: "",
   301  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2",
   302  				Config: &config.ReplicaConfig{
   303  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolAvro.String())},
   304  				},
   305  			},
   306  			expectedProtocol: config.ProtocolAvro,
   307  		},
   308  		{
   309  			info: &ChangeFeedInfo{
   310  				AdminJobType:   AdminStop,
   311  				State:          StateStopped,
   312  				Error:          nil,
   313  				CreatorVersion: "",
   314  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2",
   315  				Config: &config.ReplicaConfig{
   316  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   317  				},
   318  			},
   319  			expectedProtocol: config.ProtocolOpen,
   320  		},
   321  		{
   322  			info: &ChangeFeedInfo{
   323  				AdminJobType:   AdminStop,
   324  				State:          StateStopped,
   325  				Error:          nil,
   326  				CreatorVersion: "",
   327  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2",
   328  				Config: &config.ReplicaConfig{
   329  					Sink: &config.SinkConfig{Protocol: util.AddressOf("random")},
   330  				},
   331  			},
   332  			expectedProtocol: config.ProtocolOpen,
   333  		},
   334  		{
   335  			info: &ChangeFeedInfo{
   336  				AdminJobType:   AdminStop,
   337  				State:          StateStopped,
   338  				Error:          nil,
   339  				CreatorVersion: "5.3.0",
   340  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2",
   341  				Config: &config.ReplicaConfig{
   342  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   343  				},
   344  			},
   345  			expectedProtocol: config.ProtocolOpen,
   346  		},
   347  		{
   348  			info: &ChangeFeedInfo{
   349  				AdminJobType:   AdminStop,
   350  				State:          StateStopped,
   351  				Error:          nil,
   352  				CreatorVersion: "5.3.0",
   353  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2",
   354  				Config: &config.ReplicaConfig{
   355  					Sink: &config.SinkConfig{Protocol: util.AddressOf("random")},
   356  				},
   357  			},
   358  			expectedProtocol: config.ProtocolOpen,
   359  		},
   360  		{
   361  			info: &ChangeFeedInfo{
   362  				AdminJobType:   AdminStop,
   363  				State:          StateStopped,
   364  				Error:          nil,
   365  				CreatorVersion: "5.3.0",
   366  				SinkURI:        "mysql://127.0.0.1:9092/ticdc-test2",
   367  				Config: &config.ReplicaConfig{
   368  					Sink: &config.SinkConfig{Protocol: util.AddressOf("default")},
   369  				},
   370  			},
   371  			expectedProtocolStr: util.AddressOf(emptyProtocolStr),
   372  		},
   373  		{
   374  			info: &ChangeFeedInfo{
   375  				AdminJobType:   AdminStop,
   376  				State:          StateStopped,
   377  				Error:          nil,
   378  				CreatorVersion: "5.3.0",
   379  				SinkURI:        "tidb://127.0.0.1:9092/ticdc-test2",
   380  				Config: &config.ReplicaConfig{
   381  					Sink: &config.SinkConfig{Protocol: util.AddressOf("random")},
   382  				},
   383  			},
   384  			expectedProtocolStr: util.AddressOf(emptyProtocolStr),
   385  		},
   386  	}
   387  
   388  	for _, tc := range configTestCases {
   389  		tc.info.FixIncompatible()
   390  		if tc.expectedProtocolStr != nil {
   391  			require.Equal(t, tc.expectedProtocolStr, tc.info.Config.Sink.Protocol)
   392  		} else {
   393  			_, err := config.ParseSinkProtocolFromString(util.GetOrZero(tc.info.Config.Sink.Protocol))
   394  			if strings.Contains(tc.info.SinkURI, "kafka") {
   395  				require.NoError(t, err)
   396  			} else {
   397  				require.Error(t, err)
   398  				require.Contains(t, err.Error(), "ErrSinkUnknownProtocol")
   399  			}
   400  		}
   401  	}
   402  
   403  	sinkURITestCases := []struct {
   404  		info                *ChangeFeedInfo
   405  		expectedSinkURI     string
   406  		expectedProtocolStr *string
   407  	}{
   408  		{
   409  			info: &ChangeFeedInfo{
   410  				AdminJobType:   AdminStop,
   411  				State:          StateStopped,
   412  				Error:          nil,
   413  				CreatorVersion: "",
   414  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2?protocol=canal",
   415  				Config: &config.ReplicaConfig{
   416  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   417  				},
   418  			},
   419  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=canal",
   420  		},
   421  		{
   422  			info: &ChangeFeedInfo{
   423  				AdminJobType:   AdminStop,
   424  				State:          StateStopped,
   425  				Error:          nil,
   426  				CreatorVersion: "",
   427  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2?protocol=random",
   428  				Config: &config.ReplicaConfig{
   429  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   430  				},
   431  			},
   432  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=open-protocol",
   433  		},
   434  		{
   435  			info: &ChangeFeedInfo{
   436  				AdminJobType:   AdminStop,
   437  				State:          StateStopped,
   438  				Error:          nil,
   439  				CreatorVersion: "5.3.0",
   440  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2?protocol=canal",
   441  				Config: &config.ReplicaConfig{
   442  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   443  				},
   444  			},
   445  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=canal",
   446  		},
   447  		{
   448  			info: &ChangeFeedInfo{
   449  				AdminJobType:   AdminStop,
   450  				State:          StateStopped,
   451  				Error:          nil,
   452  				CreatorVersion: "5.3.0",
   453  				SinkURI:        "kafka://127.0.0.1:9092/ticdc-test2?protocol=random",
   454  				Config: &config.ReplicaConfig{
   455  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   456  				},
   457  			},
   458  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=open-protocol",
   459  		},
   460  		{
   461  			info: &ChangeFeedInfo{
   462  				AdminJobType:   AdminStop,
   463  				State:          StateStopped,
   464  				Error:          nil,
   465  				CreatorVersion: "5.3.0",
   466  				SinkURI:        "mysql://127.0.0.1:9092/ticdc-test2?protocol=random",
   467  				Config: &config.ReplicaConfig{
   468  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   469  				},
   470  			},
   471  			expectedSinkURI:     "mysql://127.0.0.1:9092/ticdc-test2",
   472  			expectedProtocolStr: &emptyProtocolStr,
   473  		},
   474  		{
   475  			info: &ChangeFeedInfo{
   476  				AdminJobType:   AdminStop,
   477  				State:          StateStopped,
   478  				Error:          nil,
   479  				CreatorVersion: "5.3.0",
   480  				SinkURI:        "mysql://127.0.0.1:9092/ticdc-test2?protocol=default",
   481  				Config: &config.ReplicaConfig{
   482  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolAvro.String())},
   483  				},
   484  			},
   485  			expectedSinkURI:     "mysql://127.0.0.1:9092/ticdc-test2",
   486  			expectedProtocolStr: &emptyProtocolStr,
   487  		},
   488  	}
   489  
   490  	for _, tc := range sinkURITestCases {
   491  		tc.info.FixIncompatible()
   492  		require.Equal(t, tc.expectedSinkURI, tc.info.SinkURI)
   493  		if tc.expectedProtocolStr != nil {
   494  			require.Equal(
   495  				t,
   496  				util.GetOrZero(tc.expectedProtocolStr),
   497  				util.GetOrZero(tc.info.Config.Sink.Protocol),
   498  			)
   499  		}
   500  	}
   501  }
   502  
   503  func TestFixState(t *testing.T) {
   504  	t.Parallel()
   505  
   506  	testCases := []struct {
   507  		info          *ChangeFeedInfo
   508  		expectedState FeedState
   509  	}{
   510  		{
   511  			info: &ChangeFeedInfo{
   512  				AdminJobType: AdminNone,
   513  				State:        StateNormal,
   514  				Error:        nil,
   515  			},
   516  			expectedState: StateNormal,
   517  		},
   518  		{
   519  			info: &ChangeFeedInfo{
   520  				AdminJobType: AdminResume,
   521  				State:        StateNormal,
   522  				Error:        nil,
   523  			},
   524  			expectedState: StateNormal,
   525  		},
   526  		{
   527  			info: &ChangeFeedInfo{
   528  				AdminJobType: AdminNone,
   529  				State:        StateNormal,
   530  				Error: &RunningError{
   531  					Code: string(errors.ErrClusterIDMismatch.RFCCode()),
   532  				},
   533  			},
   534  			expectedState: StateWarning,
   535  		},
   536  		{
   537  			info: &ChangeFeedInfo{
   538  				AdminJobType: AdminResume,
   539  				State:        StateNormal,
   540  				Error: &RunningError{
   541  					Code: string(errors.ErrClusterIDMismatch.RFCCode()),
   542  				},
   543  			},
   544  			expectedState: StateWarning,
   545  		},
   546  		{
   547  			info: &ChangeFeedInfo{
   548  				AdminJobType: AdminStop,
   549  				State:        StateNormal,
   550  				Error:        nil,
   551  			},
   552  			expectedState: StateStopped,
   553  		},
   554  		{
   555  			info: &ChangeFeedInfo{
   556  				AdminJobType: AdminFinish,
   557  				State:        StateNormal,
   558  				Error:        nil,
   559  			},
   560  			expectedState: StateFinished,
   561  		},
   562  		{
   563  			info: &ChangeFeedInfo{
   564  				AdminJobType: AdminRemove,
   565  				State:        StateNormal,
   566  				Error:        nil,
   567  			},
   568  			expectedState: StateRemoved,
   569  		},
   570  		{
   571  			info: &ChangeFeedInfo{
   572  				AdminJobType: AdminRemove,
   573  				State:        StateNormal,
   574  				Error:        nil,
   575  			},
   576  			expectedState: StateRemoved,
   577  		},
   578  	}
   579  
   580  	for _, tc := range testCases {
   581  		tc.info.fixState()
   582  		require.Equal(t, tc.expectedState, tc.info.State)
   583  	}
   584  }
   585  
   586  func TestFixMysqlSinkProtocol(t *testing.T) {
   587  	t.Parallel()
   588  	// Test fixing the protocol in the configuration.
   589  	configTestCases := []struct {
   590  		info             *ChangeFeedInfo
   591  		expectedProtocol *string
   592  	}{
   593  		{
   594  			info: &ChangeFeedInfo{
   595  				SinkURI: "mysql://root:test@127.0.0.1:3306/",
   596  				Config: &config.ReplicaConfig{
   597  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   598  				},
   599  			},
   600  			expectedProtocol: util.AddressOf(""),
   601  		},
   602  		{
   603  			info: &ChangeFeedInfo{
   604  				SinkURI: "mysql://root:test@127.0.0.1:3306/",
   605  				Config: &config.ReplicaConfig{
   606  					Sink: &config.SinkConfig{Protocol: util.AddressOf("whatever")},
   607  				},
   608  			},
   609  			expectedProtocol: util.AddressOf(""),
   610  		},
   611  	}
   612  
   613  	for _, tc := range configTestCases {
   614  		tc.info.fixMySQLSinkProtocol()
   615  		require.Equal(t, tc.expectedProtocol, tc.info.Config.Sink.Protocol)
   616  	}
   617  
   618  	sinkURITestCases := []struct {
   619  		info            *ChangeFeedInfo
   620  		expectedSinkURI string
   621  	}{
   622  		{
   623  			info: &ChangeFeedInfo{
   624  				SinkURI: "mysql://root:test@127.0.0.1:3306/?protocol=open-protocol",
   625  				Config: &config.ReplicaConfig{
   626  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   627  				},
   628  			},
   629  			expectedSinkURI: "mysql://root:test@127.0.0.1:3306/",
   630  		},
   631  		{
   632  			info: &ChangeFeedInfo{
   633  				SinkURI: "mysql://root:test@127.0.0.1:3306/?protocol=default",
   634  				Config: &config.ReplicaConfig{
   635  					Sink: &config.SinkConfig{Protocol: util.AddressOf("")},
   636  				},
   637  			},
   638  			expectedSinkURI: "mysql://root:test@127.0.0.1:3306/",
   639  		},
   640  	}
   641  
   642  	for _, tc := range sinkURITestCases {
   643  		tc.info.fixMySQLSinkProtocol()
   644  		require.Equal(t, tc.expectedSinkURI, tc.info.SinkURI)
   645  	}
   646  }
   647  
   648  func TestFixMQSinkProtocol(t *testing.T) {
   649  	t.Parallel()
   650  
   651  	// Test fixing the protocol in the configuration.
   652  	configTestCases := []struct {
   653  		info             *ChangeFeedInfo
   654  		expectedProtocol config.Protocol
   655  	}{
   656  		{
   657  			info: &ChangeFeedInfo{
   658  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2",
   659  				Config: &config.ReplicaConfig{
   660  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolCanal.String())},
   661  				},
   662  			},
   663  			expectedProtocol: config.ProtocolCanal,
   664  		},
   665  		{
   666  			info: &ChangeFeedInfo{
   667  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2",
   668  				Config: &config.ReplicaConfig{
   669  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   670  				},
   671  			},
   672  			expectedProtocol: config.ProtocolOpen,
   673  		},
   674  		{
   675  			info: &ChangeFeedInfo{
   676  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2",
   677  				Config: &config.ReplicaConfig{
   678  					Sink: &config.SinkConfig{Protocol: util.AddressOf("random")},
   679  				},
   680  			},
   681  			expectedProtocol: config.ProtocolOpen,
   682  		},
   683  	}
   684  
   685  	for _, tc := range configTestCases {
   686  		tc.info.fixMQSinkProtocol()
   687  		protocol, err := config.ParseSinkProtocolFromString(util.GetOrZero(tc.info.Config.Sink.Protocol))
   688  		require.Nil(t, err)
   689  		require.Equal(t, tc.expectedProtocol, protocol)
   690  	}
   691  
   692  	// Test fixing the protocol in SinkURI.
   693  	sinkURITestCases := []struct {
   694  		info            *ChangeFeedInfo
   695  		expectedSinkURI string
   696  	}{
   697  		{
   698  			info: &ChangeFeedInfo{
   699  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2",
   700  				Config: &config.ReplicaConfig{
   701  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolCanal.String())},
   702  				},
   703  			},
   704  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2",
   705  		},
   706  		{
   707  			info: &ChangeFeedInfo{
   708  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=canal",
   709  				Config: &config.ReplicaConfig{
   710  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   711  				},
   712  			},
   713  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=canal",
   714  		},
   715  		{
   716  			info: &ChangeFeedInfo{
   717  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=random",
   718  				Config: &config.ReplicaConfig{
   719  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   720  				},
   721  			},
   722  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=open-protocol",
   723  		},
   724  		{
   725  			info: &ChangeFeedInfo{
   726  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=random&max-message-bytes=15",
   727  				Config: &config.ReplicaConfig{
   728  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   729  				},
   730  			},
   731  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?max-message-bytes=15&" +
   732  				"protocol=open-protocol",
   733  		},
   734  		{
   735  			info: &ChangeFeedInfo{
   736  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=default&max-message-bytes=15",
   737  				Config: &config.ReplicaConfig{
   738  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   739  				},
   740  			},
   741  			expectedSinkURI: "kafka://127.0.0.1:9092/ticdc-test2?max-message-bytes=15&" +
   742  				"protocol=open-protocol",
   743  		},
   744  	}
   745  
   746  	for _, tc := range sinkURITestCases {
   747  		tc.info.fixMQSinkProtocol()
   748  		require.Equal(t, tc.expectedSinkURI, tc.info.SinkURI)
   749  	}
   750  }
   751  
   752  func TestFixMemoryQuotaIncompatible(t *testing.T) {
   753  	t.Parallel()
   754  
   755  	testCases := []struct {
   756  		info                *ChangeFeedInfo
   757  		expectedMemoryQuota uint64
   758  	}{
   759  		{
   760  			info: &ChangeFeedInfo{
   761  				CreatorVersion: "",
   762  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   763  				Config: &config.ReplicaConfig{
   764  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   765  				},
   766  			},
   767  			expectedMemoryQuota: config.DefaultChangefeedMemoryQuota,
   768  		},
   769  		{
   770  			info: &ChangeFeedInfo{
   771  				CreatorVersion: "6.5.0",
   772  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   773  				Config: &config.ReplicaConfig{
   774  					MemoryQuota: 0,
   775  					Sink:        &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   776  				},
   777  			},
   778  			expectedMemoryQuota: config.DefaultChangefeedMemoryQuota,
   779  		},
   780  		{
   781  			info: &ChangeFeedInfo{
   782  				CreatorVersion: "6.5.0",
   783  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   784  				Config: &config.ReplicaConfig{
   785  					MemoryQuota: 10485760,
   786  					Sink:        &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   787  				},
   788  			},
   789  			expectedMemoryQuota: 10485760,
   790  		},
   791  	}
   792  
   793  	for _, tc := range testCases {
   794  		tc.info.FixIncompatible()
   795  		require.Equal(t, tc.expectedMemoryQuota, tc.info.Config.MemoryQuota)
   796  	}
   797  }
   798  
   799  func TestFixSchedulerIncompatible(t *testing.T) {
   800  	t.Parallel()
   801  
   802  	testCases := []struct {
   803  		info              *ChangeFeedInfo
   804  		expectedScheduler *config.ChangefeedSchedulerConfig
   805  	}{
   806  		{
   807  			info: &ChangeFeedInfo{
   808  				CreatorVersion: "",
   809  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   810  				Config: &config.ReplicaConfig{
   811  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   812  				},
   813  			},
   814  			expectedScheduler: config.GetDefaultReplicaConfig().Clone().Scheduler,
   815  		},
   816  		{
   817  			info: &ChangeFeedInfo{
   818  				CreatorVersion: "6.5.0",
   819  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   820  				Config: &config.ReplicaConfig{
   821  					Scheduler: &config.ChangefeedSchedulerConfig{},
   822  					Sink:      &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   823  				},
   824  			},
   825  			expectedScheduler: &config.ChangefeedSchedulerConfig{},
   826  		},
   827  		{
   828  			info: &ChangeFeedInfo{
   829  				CreatorVersion: "6.6.0",
   830  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   831  				Config: &config.ReplicaConfig{
   832  					Scheduler: &config.ChangefeedSchedulerConfig{},
   833  					Sink:      &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   834  				},
   835  			},
   836  			expectedScheduler: &config.ChangefeedSchedulerConfig{},
   837  		},
   838  		{
   839  			info: &ChangeFeedInfo{
   840  				CreatorVersion: "6.6.0",
   841  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   842  				Config: &config.ReplicaConfig{
   843  					Scheduler: &config.ChangefeedSchedulerConfig{RegionPerSpan: 1000},
   844  					Sink:      &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   845  				},
   846  			},
   847  			expectedScheduler: &config.ChangefeedSchedulerConfig{
   848  				RegionThreshold: 1000, EnableTableAcrossNodes: true,
   849  			},
   850  		},
   851  		{
   852  			info: &ChangeFeedInfo{
   853  				CreatorVersion: "6.7.0",
   854  				SinkURI:        "mysql://root:test@127.0.0.1:3306/",
   855  				Config: &config.ReplicaConfig{
   856  					Scheduler: &config.ChangefeedSchedulerConfig{
   857  						RegionThreshold: 1000, WriteKeyThreshold: 1000,
   858  					},
   859  					Sink: &config.SinkConfig{Protocol: util.AddressOf(config.ProtocolDefault.String())},
   860  				},
   861  			},
   862  			expectedScheduler: &config.ChangefeedSchedulerConfig{
   863  				RegionThreshold: 1000, WriteKeyThreshold: 1000,
   864  			},
   865  		},
   866  	}
   867  
   868  	for _, tc := range testCases {
   869  		tc.info.FixIncompatible()
   870  		require.EqualValues(t, tc.expectedScheduler, tc.info.Config.Scheduler)
   871  	}
   872  }
   873  
   874  func TestChangeFeedInfoClone(t *testing.T) {
   875  	t.Parallel()
   876  
   877  	info := &ChangeFeedInfo{
   878  		SinkURI: "blackhole://",
   879  		StartTs: 417257993615179777,
   880  		Config: &config.ReplicaConfig{
   881  			CaseSensitive:    true,
   882  			CheckGCSafePoint: true,
   883  		},
   884  	}
   885  
   886  	cloned, err := info.Clone()
   887  	require.Nil(t, err)
   888  	sinkURI := "mysql://unix:/var/run/tidb.sock"
   889  	cloned.SinkURI = sinkURI
   890  	require.Equal(t, sinkURI, cloned.SinkURI)
   891  	require.Equal(t, "blackhole://", info.SinkURI)
   892  }
   893  
   894  func TestChangefeedInfoStringer(t *testing.T) {
   895  	t.Parallel()
   896  
   897  	testcases := []struct {
   898  		info                  *ChangeFeedInfo
   899  		expectedSinkURIRegexp string
   900  	}{
   901  		{
   902  			&ChangeFeedInfo{
   903  				SinkURI: "blackhole://",
   904  				StartTs: 418881574869139457,
   905  			},
   906  			`.*blackhole:.*`,
   907  		},
   908  		{
   909  			&ChangeFeedInfo{
   910  				SinkURI: "kafka://127.0.0.1:9092/ticdc-test2",
   911  				StartTs: 418881574869139457,
   912  			},
   913  			`.*kafka://.*ticdc-test2.*`,
   914  		},
   915  		{
   916  			&ChangeFeedInfo{
   917  				SinkURI: "mysql://root:124567@127.0.0.1:3306/",
   918  				StartTs: 418881574869139457,
   919  			},
   920  			`.*mysql://root:xxxxx@127.0.0.1:3306.*`,
   921  		},
   922  		{
   923  			&ChangeFeedInfo{
   924  				SinkURI: "mysql://root@127.0.0.1:3306/",
   925  				StartTs: 418881574869139457,
   926  			},
   927  			`.*mysql://root@127.0.0.1:3306.*`,
   928  		},
   929  		{
   930  			&ChangeFeedInfo{
   931  				SinkURI: "mysql://root:test%21%23%24%25%5E%26%2A@127.0.0.1:3306/",
   932  				StartTs: 418881574869139457,
   933  			},
   934  			`.*mysql://root:xxxxx@127.0.0.1:3306/.*`,
   935  		},
   936  	}
   937  
   938  	for _, tc := range testcases {
   939  		require.Regexp(t, tc.expectedSinkURIRegexp, tc.info.String())
   940  	}
   941  }
   942  
   943  func TestValidateChangefeedID(t *testing.T) {
   944  	t.Parallel()
   945  
   946  	tests := []struct {
   947  		name    string
   948  		id      string
   949  		wantErr bool
   950  	}{
   951  		{
   952  			name:    "alphabet",
   953  			id:      "testTtTT",
   954  			wantErr: false,
   955  		},
   956  		{
   957  			name:    "number",
   958  			id:      "01131323",
   959  			wantErr: false,
   960  		},
   961  		{
   962  			name:    "mixed",
   963  			id:      "9ff52acaA-aea6-4022-8eVc4-fbee3fD2c7890",
   964  			wantErr: false,
   965  		},
   966  		{
   967  			name:    "len==128",
   968  			id:      "1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-1234567890123456789012345678901234567890",
   969  			wantErr: false,
   970  		},
   971  		{
   972  			name:    "empty string 1",
   973  			id:      "",
   974  			wantErr: true,
   975  		},
   976  		{
   977  			name:    "empty string 2",
   978  			id:      "   ",
   979  			wantErr: true,
   980  		},
   981  		{
   982  			name:    "test_task",
   983  			id:      "test_task ",
   984  			wantErr: true,
   985  		},
   986  		{
   987  			name:    "job$",
   988  			id:      "job$ ",
   989  			wantErr: true,
   990  		},
   991  		{
   992  			name:    "test-",
   993  			id:      "test-",
   994  			wantErr: true,
   995  		},
   996  		{
   997  			name:    "-",
   998  			id:      "-",
   999  			wantErr: true,
  1000  		},
  1001  		{
  1002  			name:    "-sfsdfdf1",
  1003  			id:      "-sfsdfdf1",
  1004  			wantErr: true,
  1005  		},
  1006  		{
  1007  			name:    "len==129",
  1008  			id:      "1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-123456789012345678901234567890",
  1009  			wantErr: true,
  1010  		},
  1011  	}
  1012  	for _, tt := range tests {
  1013  		err := ValidateChangefeedID(tt.id)
  1014  		if !tt.wantErr {
  1015  			require.Nil(t, err, fmt.Sprintf("case:%s", tt.name))
  1016  		} else {
  1017  			require.True(t, errors.ErrInvalidChangefeedID.Equal(err),
  1018  				fmt.Sprintf("case:%s", tt.name))
  1019  		}
  1020  	}
  1021  }
  1022  
  1023  func TestValidateNamespace(t *testing.T) {
  1024  	t.Parallel()
  1025  
  1026  	tests := []struct {
  1027  		name    string
  1028  		id      string
  1029  		wantErr bool
  1030  	}{
  1031  		{
  1032  			name:    "alphabet",
  1033  			id:      "testTtTT",
  1034  			wantErr: false,
  1035  		},
  1036  		{
  1037  			name:    "number",
  1038  			id:      "01131323",
  1039  			wantErr: false,
  1040  		},
  1041  		{
  1042  			name:    "mixed",
  1043  			id:      "9ff52acaA-aea6-4022-8eVc4-fbee3fD2c7890",
  1044  			wantErr: false,
  1045  		},
  1046  		{
  1047  			name: "len==128",
  1048  			id: "1234567890-1234567890-1234567890-1234567890-" +
  1049  				"1234567890-1234567890-1234567890-1234567890-" +
  1050  				"1234567890123456789012345678901234567890",
  1051  			wantErr: false,
  1052  		},
  1053  		{
  1054  			name:    "empty string 1",
  1055  			id:      "",
  1056  			wantErr: true,
  1057  		},
  1058  		{
  1059  			name:    "empty string 2",
  1060  			id:      "   ",
  1061  			wantErr: true,
  1062  		},
  1063  		{
  1064  			name:    "test_task",
  1065  			id:      "test_task ",
  1066  			wantErr: true,
  1067  		},
  1068  		{
  1069  			name:    "job$",
  1070  			id:      "job$ ",
  1071  			wantErr: true,
  1072  		},
  1073  		{
  1074  			name:    "test-",
  1075  			id:      "test-",
  1076  			wantErr: true,
  1077  		},
  1078  		{
  1079  			name:    "-",
  1080  			id:      "-",
  1081  			wantErr: true,
  1082  		},
  1083  		{
  1084  			name:    "-sfsdfdf1",
  1085  			id:      "-sfsdfdf1",
  1086  			wantErr: true,
  1087  		},
  1088  		{
  1089  			name: "len==129",
  1090  			id: "1234567890-1234567890-1234567890-1234567890-1234567890-1234567890-" +
  1091  				"1234567890-1234567890-1234567890-123456789012345678901234567890",
  1092  			wantErr: true,
  1093  		},
  1094  	}
  1095  	for _, tt := range tests {
  1096  		err := ValidateNamespace(tt.id)
  1097  		if !tt.wantErr {
  1098  			require.Nil(t, err, fmt.Sprintf("case:%s", tt.name))
  1099  		} else {
  1100  			require.True(t, errors.ErrInvalidNamespace.Equal(err),
  1101  				fmt.Sprintf("case:%s", tt.name))
  1102  		}
  1103  	}
  1104  }
  1105  
  1106  func TestGetTs(t *testing.T) {
  1107  	t.Parallel()
  1108  
  1109  	var (
  1110  		startTs      uint64 = 418881574869139457
  1111  		targetTs     uint64 = 420891571239139085
  1112  		checkpointTs uint64 = 420874357546418177
  1113  		createTime          = time.Now()
  1114  		info                = &ChangeFeedInfo{
  1115  			SinkURI:    "blackhole://",
  1116  			CreateTime: createTime,
  1117  		}
  1118  	)
  1119  	require.Equal(t, info.GetStartTs(), oracle.GoTimeToTS(createTime))
  1120  	info.StartTs = startTs
  1121  	require.Equal(t, info.GetStartTs(), startTs)
  1122  
  1123  	require.Equal(t, info.GetTargetTs(), uint64(math.MaxUint64))
  1124  	info.TargetTs = targetTs
  1125  	require.Equal(t, info.GetTargetTs(), targetTs)
  1126  
  1127  	require.Equal(t, info.GetCheckpointTs(nil), startTs)
  1128  	status := &ChangeFeedStatus{CheckpointTs: checkpointTs}
  1129  	require.Equal(t, info.GetCheckpointTs(status), checkpointTs)
  1130  }