github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/engine/test/e2e/e2e_worker_exit_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package e2e_test
    15  
    16  import (
    17  	"context"
    18  	"encoding/json"
    19  	"testing"
    20  	"time"
    21  
    22  	"github.com/pingcap/log"
    23  	pb "github.com/pingcap/tiflow/engine/enginepb"
    24  	"github.com/pingcap/tiflow/engine/jobmaster/fakejob"
    25  	"github.com/pingcap/tiflow/engine/pkg/tenant"
    26  	"github.com/pingcap/tiflow/engine/test/e2e"
    27  	"github.com/stretchr/testify/require"
    28  	"go.uber.org/zap"
    29  )
    30  
    31  func TestWorkerExit(t *testing.T) {
    32  	// TODO: make the following variables configurable
    33  	var (
    34  		masterAddrs           = []string{"127.0.0.1:10245", "127.0.0.1:10246", "127.0.0.1:10247"}
    35  		businessMetaAddrs     = []string{"127.0.0.1:3336"}
    36  		etcdAddrs             = []string{"127.0.0.1:12479"}
    37  		etcdAddrsInContainer  = []string{"etcd-standalone:2379"}
    38  		defaultTimeoutForTest = 3 * time.Second
    39  	)
    40  
    41  	ctx := context.Background()
    42  	cfg := &fakejob.Config{
    43  		JobName:     "test-node-failure",
    44  		WorkerCount: 4,
    45  		// use a large enough target tick to ensure the fake job long running
    46  		TargetTick:      10000000,
    47  		EtcdWatchEnable: true,
    48  		EtcdEndpoints:   etcdAddrsInContainer,
    49  		EtcdWatchPrefix: "/fake-job/test/",
    50  
    51  		InjectErrorInterval: time.Second * 3,
    52  	}
    53  	cfgBytes, err := json.Marshal(cfg)
    54  	require.NoError(t, err)
    55  
    56  	fakeJobCfg := &e2e.FakeJobConfig{
    57  		EtcdEndpoints: etcdAddrs,
    58  		WorkerCount:   cfg.WorkerCount,
    59  		KeyPrefix:     cfg.EtcdWatchPrefix,
    60  	}
    61  
    62  	cli, err := e2e.NewUTCli(ctx, masterAddrs, businessMetaAddrs, tenant.DefaultUserProjectInfo,
    63  		fakeJobCfg)
    64  	require.NoError(t, err)
    65  
    66  	ctx1, cancel := context.WithTimeout(ctx, defaultTimeoutForTest)
    67  	defer cancel()
    68  	jobID, err := cli.CreateJob(ctx1, pb.Job_FakeJob, cfgBytes)
    69  	require.NoError(t, err)
    70  
    71  	err = cli.InitializeMetaClient(jobID)
    72  	require.NoError(t, err)
    73  
    74  	require.Eventually(t, func() bool {
    75  		ctx1, cancel := context.WithTimeout(ctx, defaultTimeoutForTest)
    76  		defer cancel()
    77  		// check tick increases to ensure all workers are online
    78  		// TODO modify the test case to use a "restart-count" as a terminating condition.
    79  		targetTick := int64(1000)
    80  		for jobIdx := 0; jobIdx < cfg.WorkerCount; jobIdx++ {
    81  			err := cli.CheckFakeJobTick(ctx1, jobID, jobIdx, targetTick)
    82  			if err != nil {
    83  				log.Warn("check fake job tick failed", zap.Error(err))
    84  				return false
    85  			}
    86  		}
    87  		return true
    88  	}, time.Second*300, time.Second*2)
    89  
    90  	ctx1, cancel = context.WithTimeout(ctx, defaultTimeoutForTest)
    91  	defer cancel()
    92  	err = cli.CancelJob(ctx1, jobID)
    93  	require.NoError(t, err)
    94  }