github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/column-mapping/column_test.go (about)

     1  // Copyright 2018 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package column
    15  
    16  import (
    17  	"fmt"
    18  	"testing"
    19  
    20  	"github.com/stretchr/testify/require"
    21  )
    22  
    23  func TestRule(t *testing.T) {
    24  	// test invalid rules
    25  	inValidRule := &Rule{"test*", "abc*", "id", "id", "Error", nil, "xxx"}
    26  	require.Error(t, inValidRule.Valid())
    27  
    28  	inValidRule.TargetColumn = ""
    29  	require.Error(t, inValidRule.Valid())
    30  
    31  	inValidRule.Expression = AddPrefix
    32  	inValidRule.TargetColumn = "id"
    33  	require.Error(t, inValidRule.Valid())
    34  
    35  	inValidRule.Arguments = []string{"1"}
    36  	require.NoError(t, inValidRule.Valid())
    37  
    38  	inValidRule.Expression = PartitionID
    39  	require.Error(t, inValidRule.Valid())
    40  
    41  	inValidRule.Arguments = []string{"1", "test_", "t_"}
    42  	require.NoError(t, inValidRule.Valid())
    43  }
    44  
    45  func TestHandle(t *testing.T) {
    46  	rules := []*Rule{
    47  		{"Test*", "xxx*", "", "id", AddPrefix, []string{"instance_id:"}, "xx"},
    48  	}
    49  
    50  	// initial column mapping
    51  	m, err := NewMapping(false, rules)
    52  	require.NoError(t, err)
    53  	require.Len(t, m.cache.infos, 0)
    54  
    55  	// test add prefix, add suffix is similar
    56  	vals, poss, err := m.HandleRowValue("test", "xxx", []string{"age", "id"}, []interface{}{1, "1"})
    57  	require.NoError(t, err)
    58  	require.Equal(t, []interface{}{1, "instance_id:1"}, vals)
    59  	require.Equal(t, []int{-1, 1}, poss)
    60  
    61  	// test cache
    62  	vals, poss, err = m.HandleRowValue("test", "xxx", []string{"name"}, []interface{}{1, "1"})
    63  	require.NoError(t, err)
    64  	require.Equal(t, []interface{}{1, "instance_id:1"}, vals)
    65  	require.Equal(t, []int{-1, 1}, poss)
    66  
    67  	// test resetCache
    68  	m.resetCache()
    69  	_, _, err = m.HandleRowValue("test", "xxx", []string{"name"}, []interface{}{"1"})
    70  
    71  	require.Error(t, err)
    72  
    73  	// test DDL
    74  	_, _, err = m.HandleDDL("test", "xxx", []string{"id", "age"}, "create table xxx")
    75  	require.Error(t, err)
    76  
    77  	statement, poss, err := m.HandleDDL("abc", "xxx", []string{"id", "age"}, "create table xxx")
    78  	require.NoError(t, err)
    79  	require.Equal(t, "create table xxx", statement)
    80  	require.Nil(t, poss)
    81  }
    82  
    83  func TestQueryColumnInfo(t *testing.T) {
    84  	SetPartitionRule(4, 7, 8)
    85  	rules := []*Rule{
    86  		{"test*", "xxx*", "", "id", PartitionID, []string{"8", "test_", "xxx_"}, "xx"},
    87  	}
    88  
    89  	// initial column mapping
    90  	m, err := NewMapping(false, rules)
    91  	require.NoError(t, err)
    92  
    93  	// test mismatch
    94  	info, err := m.queryColumnInfo("test_2", "t_1", []string{"id", "name"})
    95  	require.NoError(t, err)
    96  	require.True(t, info.ignore)
    97  
    98  	// test matched
    99  	info, err = m.queryColumnInfo("test_2", "xxx_1", []string{"id", "name"})
   100  	require.NoError(t, err)
   101  	require.Equal(t, &mappingInfo{
   102  		sourcePosition: -1,
   103  		targetPosition: 0,
   104  		rule:           rules[0],
   105  		instanceID:     int64(8 << 59),
   106  		schemaID:       int64(2 << 52),
   107  		tableID:        int64(1 << 44),
   108  	}, info)
   109  
   110  	m.resetCache()
   111  	SetPartitionRule(0, 0, 3)
   112  	info, err = m.queryColumnInfo("test_2", "xxx_1", []string{"id", "name"})
   113  	require.NoError(t, err)
   114  	require.Equal(t, &mappingInfo{
   115  		sourcePosition: -1,
   116  		targetPosition: 0,
   117  		rule:           rules[0],
   118  		instanceID:     int64(0),
   119  		schemaID:       int64(0),
   120  		tableID:        int64(1 << 60),
   121  	}, info)
   122  }
   123  
   124  func TestSetPartitionRule(t *testing.T) {
   125  	SetPartitionRule(4, 7, 8)
   126  	require.Equal(t, 4, instanceIDBitSize)
   127  	require.Equal(t, 7, schemaIDBitSize)
   128  	require.Equal(t, 8, tableIDBitSize)
   129  	require.Equal(t, int64(1<<44), maxOriginID)
   130  
   131  	SetPartitionRule(0, 3, 4)
   132  	require.Equal(t, 0, instanceIDBitSize)
   133  	require.Equal(t, 3, schemaIDBitSize)
   134  	require.Equal(t, 4, tableIDBitSize)
   135  	require.Equal(t, int64(1<<56), maxOriginID)
   136  }
   137  
   138  func TestComputePartitionID(t *testing.T) {
   139  	SetPartitionRule(4, 7, 8)
   140  
   141  	rule := &Rule{
   142  		Arguments: []string{"test", "t"},
   143  	}
   144  	_, _, _, err := computePartitionID("test_1", "t_1", rule)
   145  	require.Error(t, err)
   146  	_, _, _, err = computePartitionID("test", "t", rule)
   147  	require.Error(t, err)
   148  
   149  	rule = &Rule{
   150  		Arguments: []string{"2", "test", "t", "_"},
   151  	}
   152  	instanceID, schemaID, tableID, err := computePartitionID("test_1", "t_1", rule)
   153  	require.NoError(t, err)
   154  	require.Equal(t, int64(2<<59), instanceID)
   155  	require.Equal(t, int64(1<<52), schemaID)
   156  	require.Equal(t, int64(1<<44), tableID)
   157  
   158  	// test default partition ID to zero
   159  	instanceID, schemaID, tableID, err = computePartitionID("test", "t_3", rule)
   160  	require.NoError(t, err)
   161  	require.Equal(t, int64(2<<59), instanceID)
   162  	require.Equal(t, int64(0), schemaID)
   163  	require.Equal(t, int64(3<<44), tableID)
   164  
   165  	instanceID, schemaID, tableID, err = computePartitionID("test_5", "t", rule)
   166  	require.NoError(t, err)
   167  	require.Equal(t, int64(2<<59), instanceID)
   168  	require.Equal(t, int64(5<<52), schemaID)
   169  	require.Equal(t, int64(0), tableID)
   170  
   171  	_, _, _, err = computePartitionID("unrelated", "t_6", rule)
   172  	require.ErrorContains(t, err, "test_ is not the prefix of unrelated")
   173  
   174  	_, _, _, err = computePartitionID("test", "x", rule)
   175  	require.ErrorContains(t, err, "t_ is not the prefix of x")
   176  
   177  	_, _, _, err = computePartitionID("test_0", "t_0xa", rule)
   178  	require.ErrorContains(t, err, "the suffix of 0xa can't be converted to int64")
   179  
   180  	_, _, _, err = computePartitionID("test_0", "t_", rule)
   181  	require.ErrorContains(t, err, "t_ is not the prefix of t_") // needs a better error message
   182  
   183  	_, _, _, err = computePartitionID("testx", "t_3", rule)
   184  	require.ErrorContains(t, err, "test_ is not the prefix of testx")
   185  
   186  	SetPartitionRule(4, 0, 8)
   187  	rule = &Rule{
   188  		Arguments: []string{"2", "test_", "t_", ""},
   189  	}
   190  	instanceID, schemaID, tableID, err = computePartitionID("test_1", "t_1", rule)
   191  	require.NoError(t, err)
   192  	require.Equal(t, int64(2<<59), instanceID)
   193  	require.Equal(t, int64(0), schemaID)
   194  	require.Equal(t, int64(1<<51), tableID)
   195  
   196  	instanceID, schemaID, tableID, err = computePartitionID("test_", "t_", rule)
   197  	require.NoError(t, err)
   198  	require.Equal(t, int64(2<<59), instanceID)
   199  	require.Equal(t, int64(0), schemaID)
   200  	require.Equal(t, int64(0), tableID)
   201  
   202  	// test ignore instance ID
   203  	SetPartitionRule(4, 7, 8)
   204  	rule = &Rule{
   205  		Arguments: []string{"", "test_", "t_", ""},
   206  	}
   207  	instanceID, schemaID, tableID, err = computePartitionID("test_1", "t_1", rule)
   208  	require.NoError(t, err)
   209  	require.Equal(t, int64(0), instanceID)
   210  	require.Equal(t, int64(1<<56), schemaID)
   211  	require.Equal(t, int64(1<<48), tableID)
   212  
   213  	// test ignore schema ID
   214  	rule = &Rule{
   215  		Arguments: []string{"2", "", "t_", ""},
   216  	}
   217  	instanceID, schemaID, tableID, err = computePartitionID("test_1", "t_1", rule)
   218  	require.NoError(t, err)
   219  	require.Equal(t, int64(2<<59), instanceID)
   220  	require.Equal(t, int64(0), schemaID)
   221  	require.Equal(t, int64(1<<51), tableID)
   222  
   223  	// test ignore schema ID
   224  	rule = &Rule{
   225  		Arguments: []string{"2", "test_", "", ""},
   226  	}
   227  	instanceID, schemaID, tableID, err = computePartitionID("test_1", "t_1", rule)
   228  	require.NoError(t, err)
   229  	require.Equal(t, int64(2<<59), instanceID)
   230  	require.Equal(t, int64(1<<52), schemaID)
   231  	require.Equal(t, int64(0), tableID)
   232  }
   233  
   234  func TestPartitionID(t *testing.T) {
   235  	SetPartitionRule(4, 7, 8)
   236  	info := &mappingInfo{
   237  		instanceID:     int64(2 << 59),
   238  		schemaID:       int64(1 << 52),
   239  		tableID:        int64(1 << 44),
   240  		targetPosition: 1,
   241  	}
   242  
   243  	// test wrong type
   244  	_, err := partitionID(info, []interface{}{1, "ha"})
   245  	require.Error(t, err)
   246  
   247  	// test exceed maxOriginID
   248  	_, err = partitionID(info, []interface{}{"ha", 1 << 44})
   249  	require.Error(t, err)
   250  
   251  	vals, err := partitionID(info, []interface{}{"ha", 1})
   252  	require.NoError(t, err)
   253  	require.Equal(t, []interface{}{"ha", int64(2<<59 | 1<<52 | 1<<44 | 1)}, vals)
   254  
   255  	info.instanceID = 0
   256  	vals, err = partitionID(info, []interface{}{"ha", "123"})
   257  	require.NoError(t, err)
   258  	require.Equal(t, []interface{}{"ha", fmt.Sprintf("%d", int64(1<<52|1<<44|123))}, vals)
   259  }
   260  
   261  func TestCaseSensitive(t *testing.T) {
   262  	// we test case insensitive in TestHandle
   263  	rules := []*Rule{
   264  		{"Test*", "xxx*", "", "id", AddPrefix, []string{"instance_id:"}, "xx"},
   265  	}
   266  
   267  	// case sensitive
   268  	// initial column mapping
   269  	m, err := NewMapping(true, rules)
   270  	require.NoError(t, err)
   271  	require.Len(t, m.cache.infos, 0)
   272  
   273  	// test add prefix, add suffix is similar
   274  	vals, poss, err := m.HandleRowValue("test", "xxx", []string{"age", "id"}, []interface{}{1, "1"})
   275  	require.NoError(t, err)
   276  	require.Equal(t, []interface{}{1, "1"}, vals)
   277  	require.Nil(t, poss)
   278  }