github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/pkg/ha/keepalive_test.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package ha
    15  
    16  import (
    17  	"context"
    18  	"sort"
    19  	"strconv"
    20  	"sync/atomic"
    21  	"time"
    22  
    23  	. "github.com/pingcap/check"
    24  	"github.com/pingcap/tiflow/dm/pkg/utils"
    25  )
    26  
    27  // keepAliveTTL is set to 0 because the actual ttl is set to minLeaseTTL of etcd
    28  // minLeaseTTL is 1 in etcd cluster.
    29  var keepAliveTTL = int64(0)
    30  
    31  func (t *testForEtcd) TestWorkerKeepAlive(c *C) {
    32  	defer clearTestInfoOperation(c)
    33  	wwm, rev, err := GetKeepAliveWorkers(etcdTestCli)
    34  	c.Assert(err, IsNil)
    35  	c.Assert(wwm, HasLen, 0)
    36  
    37  	ctx, cancel := context.WithCancel(context.Background())
    38  	defer cancel()
    39  
    40  	timeout := 2 * time.Second
    41  	evCh := make(chan WorkerEvent, 10)
    42  	errCh := make(chan error, 10)
    43  	closed := make(chan struct{})
    44  	finished := int32(0)
    45  
    46  	go func() {
    47  		WatchWorkerEvent(ctx, etcdTestCli, rev, evCh, errCh)
    48  		close(closed)
    49  	}()
    50  
    51  	cancels := make([]context.CancelFunc, 0, 5)
    52  	for i := 1; i <= 5; i++ {
    53  		worker := "worker" + strconv.Itoa(i)
    54  		curTime := time.Now()
    55  		ctx1, cancel1 := context.WithCancel(ctx)
    56  		cancels = append(cancels, cancel1)
    57  		go func(ctx context.Context) {
    58  			err1 := KeepAlive(ctx, etcdTestCli, worker, keepAliveTTL)
    59  			c.Assert(err1, IsNil, Commentf("if \"context canceled\", retry later\ncause: context used in `KeepAlive` exceed timeout of 10s (`etcdutil.DefaultRequestTimeout`)"))
    60  			atomic.AddInt32(&finished, 1)
    61  		}(ctx1)
    62  
    63  		select {
    64  		case ev := <-evCh:
    65  			c.Assert(ev.IsDeleted, IsFalse)
    66  			c.Assert(ev.WorkerName, Equals, worker)
    67  			c.Assert(ev.JoinTime.Before(curTime), IsFalse)
    68  		case <-time.After(timeout):
    69  			c.Fatal("fail to receive put ev " + strconv.Itoa(i) + " before timeout")
    70  		}
    71  	}
    72  
    73  	for i, cancel1 := range cancels {
    74  		worker := "worker" + strconv.Itoa(i+1)
    75  		cancel1()
    76  		select {
    77  		case ev := <-evCh:
    78  			c.Assert(ev.IsDeleted, IsTrue)
    79  			c.Assert(ev.WorkerName, Equals, worker)
    80  		case <-time.After(timeout):
    81  			c.Fatal("fail to receive delete ev " + strconv.Itoa(i+1) + " before timeout")
    82  		}
    83  	}
    84  
    85  	waitKeepAliveQuit := utils.WaitSomething(100, timeout, func() bool {
    86  		return atomic.LoadInt32(&finished) == 5
    87  	})
    88  	c.Assert(waitKeepAliveQuit, IsTrue)
    89  
    90  	cancel()
    91  	select {
    92  	case <-closed:
    93  	case <-time.After(timeout):
    94  		c.Fatal("fail to quit WatchWorkerEvent before timeout")
    95  	}
    96  	c.Assert(errCh, HasLen, 0)
    97  }
    98  
    99  func (t *testForEtcd) TestKeepAliveRevokeLease(c *C) {
   100  	defer clearTestInfoOperation(c)
   101  	wwm, rev, err := GetKeepAliveWorkers(etcdTestCli)
   102  	c.Assert(err, IsNil)
   103  	c.Assert(wwm, HasLen, 0)
   104  
   105  	ctx, cancel := context.WithCancel(context.Background())
   106  	defer cancel()
   107  	var (
   108  		finished         = int32(0)
   109  		hugeKeepAliveTTL = int64(100) // use a huge keepalive ttl to test whether keepalive can quit quickly
   110  		evCh             = make(chan WorkerEvent, 110)
   111  		errCh            = make(chan error, 110)
   112  		putEvent         = make([]string, 0, 50)
   113  		deleteEvent      = make([]string, 0, 50)
   114  		workerSet        = make([]string, 0, 50)
   115  	)
   116  
   117  	for i := 1; i <= 50; i++ {
   118  		worker := "worker" + strconv.Itoa(i)
   119  		workerSet = append(workerSet, worker)
   120  		ctx1, cancel1 := context.WithTimeout(ctx, 2*time.Second)
   121  		go func(ctx context.Context, cancel1 context.CancelFunc) {
   122  			err1 := KeepAlive(ctx, etcdTestCli, worker, hugeKeepAliveTTL)
   123  			cancel1()
   124  			c.Assert(err1, IsNil)
   125  			atomic.AddInt32(&finished, 1)
   126  		}(ctx1, cancel1)
   127  	}
   128  
   129  	ctx1, cancel1 := context.WithTimeout(ctx, 10*time.Second)
   130  	WatchWorkerEvent(ctx1, etcdTestCli, rev, evCh, errCh)
   131  	cancel1()
   132  	c.Assert(evCh, HasLen, 100)
   133  	c.Assert(errCh, HasLen, 0)
   134  	c.Assert(atomic.LoadInt32(&finished), Equals, int32(50))
   135  	for len(evCh) > 0 {
   136  		ev := <-evCh
   137  		if ev.IsDeleted {
   138  			deleteEvent = append(deleteEvent, ev.WorkerName)
   139  		} else {
   140  			putEvent = append(putEvent, ev.WorkerName)
   141  		}
   142  	}
   143  	sort.Strings(putEvent)
   144  	sort.Strings(deleteEvent)
   145  	sort.Strings(workerSet)
   146  	c.Assert(putEvent, DeepEquals, workerSet)
   147  	c.Assert(deleteEvent, DeepEquals, workerSet)
   148  }