github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/cdc/processor/pipeline/mounter_test.go (about)

     1  // Copyright 2021 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package pipeline
    15  
    16  import (
    17  	"context"
    18  	"errors"
    19  	"sync"
    20  	"sync/atomic"
    21  	"time"
    22  
    23  	"github.com/pingcap/check"
    24  	"github.com/pingcap/log"
    25  	"github.com/pingcap/ticdc/cdc/model"
    26  	cdcContext "github.com/pingcap/ticdc/pkg/context"
    27  	"github.com/pingcap/ticdc/pkg/pipeline"
    28  	"github.com/pingcap/ticdc/pkg/retry"
    29  	"github.com/pingcap/ticdc/pkg/util/testleak"
    30  	"go.uber.org/zap"
    31  )
    32  
    33  type mounterNodeSuite struct{}
    34  
    35  var _ = check.Suite(&mounterNodeSuite{})
    36  
    37  type checkNode struct {
    38  	c        *check.C
    39  	count    int
    40  	expected int
    41  }
    42  
    43  func (n *checkNode) Init(ctx pipeline.NodeContext) error {
    44  	// do nothing
    45  	return nil
    46  }
    47  
    48  func (n *checkNode) Receive(ctx pipeline.NodeContext) error {
    49  	message := ctx.Message()
    50  	if message.Tp == pipeline.MessageTypePolymorphicEvent {
    51  		if message.PolymorphicEvent.RawKV.OpType == model.OpTypeResolved {
    52  			n.c.Assert(n.count, check.Equals, n.expected)
    53  			return errors.New("finished")
    54  		}
    55  		n.c.Assert(message.PolymorphicEvent.Row, check.NotNil)
    56  	}
    57  
    58  	if n.count%100 == 0 {
    59  		log.Info("message received", zap.Int("count", n.count))
    60  	}
    61  
    62  	if n.count == basicsTestMessageCount/2 {
    63  		log.Info("sleeping for 5 seconds to simulate blocking")
    64  		time.Sleep(time.Second * 5)
    65  	}
    66  	n.count++
    67  	return nil
    68  }
    69  
    70  func (n *checkNode) Destroy(ctx pipeline.NodeContext) error {
    71  	return nil
    72  }
    73  
    74  const (
    75  	basicsTestMessageCount = 10000
    76  )
    77  
    78  func generateMockRawKV(ts uint64) *model.RawKVEntry {
    79  	return &model.RawKVEntry{
    80  		OpType:   model.OpTypePut,
    81  		Key:      []byte{},
    82  		Value:    []byte{},
    83  		OldValue: nil,
    84  		StartTs:  ts - 5,
    85  		CRTs:     ts,
    86  		RegionID: 0,
    87  	}
    88  }
    89  
    90  func (s *mounterNodeSuite) TestMounterNodeBasics(c *check.C) {
    91  	defer testleak.AfterTest(c)()
    92  
    93  	ctx, cancel := cdcContext.WithCancel(cdcContext.NewBackendContext4Test(false))
    94  	defer cancel()
    95  
    96  	ctx = cdcContext.WithErrorHandler(ctx, func(err error) error {
    97  		return nil
    98  	})
    99  	runnersSize, outputChannelSize := 2, 64
   100  	p := pipeline.NewPipeline(ctx, 0, runnersSize, outputChannelSize)
   101  	mounterNode := newMounterNode()
   102  	p.AppendNode(ctx, "mounter", mounterNode)
   103  
   104  	checkNode := &checkNode{
   105  		c:        c,
   106  		count:    0,
   107  		expected: basicsTestMessageCount,
   108  	}
   109  	p.AppendNode(ctx, "check", checkNode)
   110  
   111  	var sentCount int64
   112  	sendMsg := func(p *pipeline.Pipeline, msg pipeline.Message) {
   113  		err := retry.Do(context.Background(), func() error {
   114  			return p.SendToFirstNode(msg)
   115  		}, retry.WithBackoffBaseDelay(10), retry.WithBackoffMaxDelay(60*1000), retry.WithMaxTries(100))
   116  		atomic.AddInt64(&sentCount, 1)
   117  		c.Assert(err, check.IsNil)
   118  	}
   119  
   120  	mockMounterInput := make(chan *model.PolymorphicEvent, 10240)
   121  	var wg sync.WaitGroup
   122  	wg.Add(1)
   123  	go func() {
   124  		defer wg.Done()
   125  		for i := 0; i < basicsTestMessageCount; i++ {
   126  			var msg pipeline.Message
   127  			if i%100 == 0 {
   128  				// generates a control message
   129  				msg = pipeline.TickMessage()
   130  			} else {
   131  				msg = pipeline.PolymorphicEventMessage(model.NewPolymorphicEvent(generateMockRawKV(uint64(i << 5))))
   132  				msg.PolymorphicEvent.SetUpFinishedChan()
   133  				select {
   134  				case <-ctx.Done():
   135  					return
   136  				case mockMounterInput <- msg.PolymorphicEvent:
   137  				}
   138  			}
   139  			sendMsg(p, msg)
   140  		}
   141  		msg := pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, (basicsTestMessageCount<<5)+1))
   142  		sendMsg(p, msg)
   143  		c.Assert(atomic.LoadInt64(&sentCount), check.Equals, int64(basicsTestMessageCount+1))
   144  		log.Info("finished sending")
   145  	}()
   146  
   147  	wg.Add(1)
   148  	go func() {
   149  		defer wg.Done()
   150  		for {
   151  			select {
   152  			case <-ctx.Done():
   153  				return
   154  			case event := <-mockMounterInput:
   155  				event.Row = &model.RowChangedEvent{} // mocked row
   156  				event.PrepareFinished()
   157  			}
   158  		}
   159  	}()
   160  
   161  	p.Wait()
   162  	cancel()
   163  	wg.Wait()
   164  }