github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/entry/mounter_group.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package entry
    15  
    16  import (
    17  	"context"
    18  	"time"
    19  
    20  	"github.com/pingcap/errors"
    21  	"github.com/pingcap/tiflow/cdc/model"
    22  	"github.com/pingcap/tiflow/pkg/filter"
    23  	"github.com/pingcap/tiflow/pkg/integrity"
    24  	"github.com/pingcap/tiflow/pkg/util"
    25  	"golang.org/x/sync/errgroup"
    26  )
    27  
    28  // MounterGroup is a group of mounter workers
    29  type MounterGroup interface {
    30  	util.Runnable
    31  
    32  	AddEvent(ctx context.Context, event *model.PolymorphicEvent) error
    33  	TryAddEvent(ctx context.Context, event *model.PolymorphicEvent) (bool, error)
    34  }
    35  
    36  type mounterGroup struct {
    37  	schemaStorage SchemaStorage
    38  	inputCh       chan *model.PolymorphicEvent
    39  	tz            *time.Location
    40  	filter        filter.Filter
    41  	integrity     *integrity.Config
    42  
    43  	workerNum int
    44  
    45  	changefeedID model.ChangeFeedID
    46  }
    47  
    48  const (
    49  	defaultMounterWorkerNum = 16
    50  	defaultInputChanSize    = 1024
    51  	defaultMetricInterval   = 15 * time.Second
    52  )
    53  
    54  // NewMounterGroup return a group of mounters.
    55  func NewMounterGroup(
    56  	schemaStorage SchemaStorage,
    57  	workerNum int,
    58  	filter filter.Filter,
    59  	tz *time.Location,
    60  	changefeedID model.ChangeFeedID,
    61  	integrity *integrity.Config,
    62  ) *mounterGroup {
    63  	if workerNum <= 0 {
    64  		workerNum = defaultMounterWorkerNum
    65  	}
    66  	return &mounterGroup{
    67  		schemaStorage: schemaStorage,
    68  		inputCh:       make(chan *model.PolymorphicEvent, defaultInputChanSize),
    69  		filter:        filter,
    70  		tz:            tz,
    71  
    72  		integrity: integrity,
    73  
    74  		workerNum: workerNum,
    75  
    76  		changefeedID: changefeedID,
    77  	}
    78  }
    79  
    80  func (m *mounterGroup) Run(ctx context.Context, _ ...chan<- error) error {
    81  	defer func() {
    82  		mounterGroupInputChanSizeGauge.DeleteLabelValues(m.changefeedID.Namespace, m.changefeedID.ID)
    83  	}()
    84  	g, ctx := errgroup.WithContext(ctx)
    85  	for i := 0; i < m.workerNum; i++ {
    86  		g.Go(func() error {
    87  			return m.runWorker(ctx)
    88  		})
    89  	}
    90  	g.Go(func() error {
    91  		metrics := mounterGroupInputChanSizeGauge.WithLabelValues(m.changefeedID.Namespace, m.changefeedID.ID)
    92  		ticker := time.NewTicker(defaultMetricInterval)
    93  		defer ticker.Stop()
    94  		for {
    95  			select {
    96  			case <-ctx.Done():
    97  				return errors.Trace(ctx.Err())
    98  			case <-ticker.C:
    99  				metrics.Set(float64(len(m.inputCh)))
   100  			}
   101  		}
   102  	})
   103  	return g.Wait()
   104  }
   105  
   106  func (m *mounterGroup) WaitForReady(_ context.Context) {}
   107  
   108  func (m *mounterGroup) Close() {}
   109  
   110  func (m *mounterGroup) runWorker(ctx context.Context) error {
   111  	mounter := NewMounter(m.schemaStorage, m.changefeedID, m.tz, m.filter, m.integrity)
   112  	for {
   113  		select {
   114  		case <-ctx.Done():
   115  			return errors.Trace(ctx.Err())
   116  		case pEvent := <-m.inputCh:
   117  			if pEvent.RawKV.OpType == model.OpTypeResolved {
   118  				pEvent.MarkFinished()
   119  				continue
   120  			}
   121  			err := mounter.DecodeEvent(ctx, pEvent)
   122  			if err != nil {
   123  				return errors.Trace(err)
   124  			}
   125  			pEvent.MarkFinished()
   126  		}
   127  	}
   128  }
   129  
   130  func (m *mounterGroup) AddEvent(ctx context.Context, event *model.PolymorphicEvent) error {
   131  	select {
   132  	case <-ctx.Done():
   133  		return ctx.Err()
   134  	case m.inputCh <- event:
   135  		return nil
   136  	}
   137  }
   138  
   139  func (m *mounterGroup) TryAddEvent(ctx context.Context, event *model.PolymorphicEvent) (bool, error) {
   140  	select {
   141  	case <-ctx.Done():
   142  		return false, ctx.Err()
   143  	case m.inputCh <- event:
   144  		return true, nil
   145  	default:
   146  		return false, nil
   147  	}
   148  }
   149  
   150  // MockMountGroup is used for tests.
   151  type MockMountGroup struct {
   152  	IsFull bool
   153  }
   154  
   155  // Run implements util.Runnable.
   156  func (m *MockMountGroup) Run(ctx context.Context, _ ...chan<- error) error {
   157  	return nil
   158  }
   159  
   160  // WaitForReady implements util.Runnable.
   161  func (m *MockMountGroup) WaitForReady(_ context.Context) {}
   162  
   163  // Close implements util.Runnable.
   164  func (m *MockMountGroup) Close() {}
   165  
   166  // AddEvent implements MountGroup.
   167  func (m *MockMountGroup) AddEvent(ctx context.Context, event *model.PolymorphicEvent) error {
   168  	event.MarkFinished()
   169  	return nil
   170  }
   171  
   172  // TryAddEvent implements MountGroup.
   173  func (m *MockMountGroup) TryAddEvent(ctx context.Context, event *model.PolymorphicEvent) (bool, error) {
   174  	if !m.IsFull {
   175  		event.MarkFinished()
   176  		return true, nil
   177  	}
   178  	return false, nil
   179  }