github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/orchestrator/batch.go (about)

     1  // Copyright 2021 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package orchestrator
    15  
    16  import (
    17  	"github.com/pingcap/errors"
    18  	cerrors "github.com/pingcap/tiflow/pkg/errors"
    19  	"github.com/pingcap/tiflow/pkg/orchestrator/util"
    20  )
    21  
    22  const (
    23  	// 1.25 MiB
    24  	// Ref: https://etcd.io/docs/v3.3/dev-guide/limit/
    25  	etcdTxnMaxSize = 1024 * (1024 + 256)
    26  	// Ref: https://etcd.io/docs/v3.3/op-guide/configuration/#--max-txn-ops
    27  	etcdTxnMaxOps = 128
    28  )
    29  
    30  // getBatchChangedState has 4 return values:
    31  // 1.batchChangedSate
    32  // 2.number of patch apply to batchChangedState
    33  // 3.size of batchChangedState in byte
    34  // 4.error
    35  func getBatchChangedState(state map[util.EtcdKey][]byte, patchGroups [][]DataPatch) (map[util.EtcdKey][]byte, int, int, error) {
    36  	num := 0
    37  	totalSize := 0
    38  	// store changedState of multiple changefeed
    39  	batchChangedState := make(map[util.EtcdKey][]byte)
    40  	for i, patches := range patchGroups {
    41  		changedState, changedSize, err := getChangedState(state, patches)
    42  		if err != nil {
    43  			return nil, 0, 0, err
    44  		}
    45  		// if a changefeed's changedState size is larger than etcdTxnMaxSize
    46  		// or the length of changedState is larger than etcdTxnMaxOps
    47  		// we should return an error instantly
    48  		if i == 0 {
    49  			if changedSize > etcdTxnMaxSize {
    50  				return nil, 0, 0, cerrors.ErrEtcdTxnSizeExceed.GenWithStackByArgs(changedSize, etcdTxnMaxSize)
    51  			}
    52  			if len(changedState) > etcdTxnMaxOps {
    53  				return nil, 0, 0, cerrors.ErrEtcdTxnOpsExceed.GenWithStackByArgs(len(changedState), etcdTxnMaxOps)
    54  			}
    55  		}
    56  
    57  		// batchChangedState size should not exceeds the etcdTxnMaxSize limit
    58  		// and keys numbers should not exceeds the etcdTxnMaxOps limit
    59  		if totalSize+changedSize >= etcdTxnMaxSize ||
    60  			len(batchChangedState)+len(changedState) >= etcdTxnMaxOps {
    61  			break
    62  		}
    63  		for k, v := range changedState {
    64  			batchChangedState[k] = v
    65  		}
    66  		num++
    67  		totalSize += changedSize
    68  	}
    69  	return batchChangedState, num, totalSize, nil
    70  }
    71  
    72  func getChangedState(state map[util.EtcdKey][]byte, patches []DataPatch) (map[util.EtcdKey][]byte, int, error) {
    73  	changedSet := make(map[util.EtcdKey]struct{})
    74  	changeState := make(map[util.EtcdKey][]byte)
    75  	changedSize := 0
    76  	for _, patch := range patches {
    77  		err := patch.Patch(state, changedSet)
    78  		if err != nil {
    79  			if cerrors.ErrEtcdIgnore.Equal(errors.Cause(err)) {
    80  				continue
    81  			}
    82  			return nil, 0, errors.Trace(err)
    83  		}
    84  	}
    85  	for k := range changedSet {
    86  		v := state[k]
    87  		changedSize += len(k.String())*2 + len(v)
    88  		changeState[k] = v
    89  	}
    90  	return changeState, changedSize, nil
    91  }