github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/craft/craft_encoder.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License")
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.orglicensesLICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package craft
    15  
    16  import (
    17  	"context"
    18  
    19  	"github.com/pingcap/tiflow/cdc/model"
    20  	"github.com/pingcap/tiflow/pkg/config"
    21  	"github.com/pingcap/tiflow/pkg/sink/codec"
    22  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    23  )
    24  
    25  // BatchEncoder encodes the events into the byte of a batch into craft binary format.
    26  type BatchEncoder struct {
    27  	rowChangedBuffer *RowChangedEventBuffer
    28  	messageBuf       []*common.Message
    29  	callbackBuf      []func()
    30  
    31  	config *common.Config
    32  
    33  	allocator *SliceAllocator
    34  }
    35  
    36  // EncodeCheckpointEvent implements the RowEventEncoder interface
    37  func (e *BatchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) {
    38  	return common.NewResolvedMsg(
    39  		config.ProtocolCraft, nil,
    40  		NewResolvedEventEncoder(e.allocator, ts).Encode(), ts), nil
    41  }
    42  
    43  // AppendRowChangedEvent implements the RowEventEncoder interface
    44  func (e *BatchEncoder) AppendRowChangedEvent(
    45  	_ context.Context,
    46  	_ string,
    47  	ev *model.RowChangedEvent,
    48  	callback func(),
    49  ) error {
    50  	rows, size := e.rowChangedBuffer.AppendRowChangedEvent(ev, e.config.DeleteOnlyHandleKeyColumns)
    51  	if callback != nil {
    52  		e.callbackBuf = append(e.callbackBuf, callback)
    53  	}
    54  	if size > e.config.MaxMessageBytes || rows >= e.config.MaxBatchSize {
    55  		e.flush()
    56  	}
    57  	return nil
    58  }
    59  
    60  // EncodeDDLEvent implements the RowEventEncoder interface
    61  func (e *BatchEncoder) EncodeDDLEvent(ev *model.DDLEvent) (*common.Message, error) {
    62  	return common.NewDDLMsg(config.ProtocolCraft,
    63  		nil, NewDDLEventEncoder(e.allocator, ev).Encode(), ev), nil
    64  }
    65  
    66  // Build implements the RowEventEncoder interface
    67  func (e *BatchEncoder) Build() []*common.Message {
    68  	if e.rowChangedBuffer.Size() > 0 {
    69  		// flush buffered data to message buffer
    70  		e.flush()
    71  	}
    72  	ret := e.messageBuf
    73  	e.messageBuf = make([]*common.Message, 0, 2)
    74  	return ret
    75  }
    76  
    77  func (e *BatchEncoder) flush() {
    78  	headers := e.rowChangedBuffer.GetHeaders()
    79  	ts := headers.GetTs(0)
    80  	schema := headers.GetSchema(0)
    81  	table := headers.GetTable(0)
    82  	rowsCnt := e.rowChangedBuffer.RowsCount()
    83  	message := common.NewMsg(config.ProtocolCraft,
    84  		nil, e.rowChangedBuffer.Encode(), ts, model.MessageTypeRow, &schema, &table)
    85  	message.SetRowsCount(rowsCnt)
    86  	if len(e.callbackBuf) != 0 && len(e.callbackBuf) == rowsCnt {
    87  		callbacks := e.callbackBuf
    88  		message.Callback = func() {
    89  			for _, cb := range callbacks {
    90  				cb()
    91  			}
    92  		}
    93  		e.callbackBuf = make([]func(), 0)
    94  	}
    95  	e.messageBuf = append(e.messageBuf, message)
    96  }
    97  
    98  // NewBatchEncoder creates a new BatchEncoder.
    99  func NewBatchEncoder(config *common.Config) codec.RowEventEncoder {
   100  	// 64 is a magic number that come up with these assumptions and manual benchmark.
   101  	// 1. Most table will not have more than 64 columns
   102  	// 2. It only worth allocating slices in batch for slices that's small enough
   103  	return NewBatchEncoderWithAllocator(NewSliceAllocator(64), config)
   104  }
   105  
   106  type batchEncoderBuilder struct {
   107  	config *common.Config
   108  }
   109  
   110  // Build a BatchEncoder
   111  func (b *batchEncoderBuilder) Build() codec.RowEventEncoder {
   112  	return NewBatchEncoder(b.config)
   113  }
   114  
   115  // CleanMetrics do nothing
   116  func (b *batchEncoderBuilder) CleanMetrics() {}
   117  
   118  // NewBatchEncoderBuilder creates a craft batchEncoderBuilder.
   119  func NewBatchEncoderBuilder(config *common.Config) codec.RowEventEncoderBuilder {
   120  	return &batchEncoderBuilder{config: config}
   121  }
   122  
   123  // NewBatchEncoderWithAllocator creates a new BatchEncoder with given allocator.
   124  func NewBatchEncoderWithAllocator(allocator *SliceAllocator, config *common.Config) codec.RowEventEncoder {
   125  	return &BatchEncoder{
   126  		allocator:        allocator,
   127  		messageBuf:       make([]*common.Message, 0, 2),
   128  		callbackBuf:      make([]func(), 0),
   129  		rowChangedBuffer: NewRowChangedEventBuffer(allocator),
   130  		config:           config,
   131  	}
   132  }