github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/jobmaster/dm/config/config_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package config
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"testing"
    20  
    21  	"github.com/BurntSushi/toml"
    22  	dmconfig "github.com/pingcap/tiflow/dm/config"
    23  	dmmaster "github.com/pingcap/tiflow/dm/master"
    24  	"github.com/stretchr/testify/require"
    25  	"go.uber.org/atomic"
    26  )
    27  
    28  const (
    29  	jobTemplatePath    = "./job_template.yaml"
    30  	subtaskTemplateDir = "."
    31  )
    32  
    33  func checkAndNoAdjustSourceConfigMock(ctx context.Context, cfg *dmconfig.SourceConfig) error {
    34  	if _, err := cfg.Yaml(); err != nil {
    35  		return err
    36  	}
    37  	return cfg.Verify()
    38  }
    39  
    40  func TestJobCfg(t *testing.T) {
    41  	funcBackup := dmmaster.CheckAndAdjustSourceConfigFunc
    42  	dmmaster.CheckAndAdjustSourceConfigFunc = checkAndNoAdjustSourceConfigMock
    43  	defer func() {
    44  		dmmaster.CheckAndAdjustSourceConfigFunc = funcBackup
    45  	}()
    46  
    47  	jobCfg := &JobCfg{}
    48  	require.NoError(t, jobCfg.DecodeFile(jobTemplatePath))
    49  	require.Equal(t, dmconfig.ModeAll, jobCfg.TaskMode)
    50  	content, err := jobCfg.Yaml()
    51  	require.NoError(t, err)
    52  
    53  	clone, err := jobCfg.Clone()
    54  	require.NoError(t, err)
    55  	content2, err := clone.Yaml()
    56  	require.NoError(t, err)
    57  	require.Equal(t, content2, content)
    58  
    59  	dmTaskCfg, err := clone.toDMTaskConfig()
    60  	require.NoError(t, err)
    61  	require.NoError(t, clone.fromDMTaskConfig(dmTaskCfg))
    62  	content3, err := clone.Yaml()
    63  	require.NoError(t, err)
    64  	require.Equal(t, content3, content)
    65  
    66  	require.Error(t, jobCfg.DecodeFile("./job_not_exist.yaml"))
    67  	jobCfg.Upstreams[0].SourceID = ""
    68  	require.EqualError(t, jobCfg.adjust(), "source-id of 1st upstream is empty")
    69  	jobCfg.Upstreams[0].SourceID = jobCfg.Upstreams[1].SourceID
    70  	require.EqualError(t, jobCfg.adjust(), fmt.Sprintf("source-id %s is duplicated", jobCfg.Upstreams[0].SourceID))
    71  }
    72  
    73  func TestTaskCfg(t *testing.T) {
    74  	funcBackup := dmmaster.CheckAndAdjustSourceConfigFunc
    75  	dmmaster.CheckAndAdjustSourceConfigFunc = checkAndNoAdjustSourceConfigMock
    76  	defer func() {
    77  		dmmaster.CheckAndAdjustSourceConfigFunc = funcBackup
    78  	}()
    79  
    80  	jobCfg := &JobCfg{}
    81  	require.NoError(t, jobCfg.DecodeFile(jobTemplatePath))
    82  	// test update job
    83  	jobCfg.ModRevision = 1
    84  
    85  	taskCfgs := jobCfg.ToTaskCfgs()
    86  
    87  	taskCfgList := make([]*TaskCfg, 0, len(taskCfgs))
    88  	for _, taskCfg := range taskCfgs {
    89  		taskCfgList = append(taskCfgList, taskCfg)
    90  	}
    91  	jobCfg2 := FromTaskCfgs(taskCfgList)
    92  	taskCfgs = jobCfg2.ToTaskCfgs()
    93  
    94  	require.Equal(t, jobCfg.ModRevision, jobCfg2.ModRevision)
    95  
    96  	for _, taskCfg := range taskCfgs {
    97  		subTaskCfg := taskCfg.ToDMSubTaskCfg("test")
    98  		expectCfg := &dmconfig.SubTaskConfig{}
    99  		_, err := toml.DecodeFile(fmt.Sprintf("%s/dm_subtask_%d.toml", subtaskTemplateDir, taskCfg.Upstreams[0].DBCfg.Port), expectCfg)
   100  		require.NoError(t, err)
   101  		expectCfg.IOTotalBytes = atomic.NewUint64(0)
   102  		expectCfg.DumpIOTotalBytes = atomic.NewUint64(0)
   103  		// require uuid is set
   104  		require.Greater(t, len(subTaskCfg.UUID), 0)
   105  		require.Greater(t, len(subTaskCfg.DumpUUID), 0)
   106  		// don't check uuid's value
   107  		expectCfg.UUID = subTaskCfg.UUID
   108  		expectCfg.DumpUUID = subTaskCfg.DumpUUID
   109  		expectCfg.IgnoreCheckingItems = append(taskCfg.IgnoreCheckingItems, dmconfig.MetaPositionChecking)
   110  		require.EqualValues(t, expectCfg, subTaskCfg)
   111  	}
   112  }