github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/maxwell/maxwell_encoder.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package maxwell
    15  
    16  import (
    17  	"bytes"
    18  	"context"
    19  	"encoding/binary"
    20  
    21  	"github.com/pingcap/errors"
    22  	"github.com/pingcap/tiflow/cdc/model"
    23  	"github.com/pingcap/tiflow/pkg/config"
    24  	"github.com/pingcap/tiflow/pkg/sink/codec"
    25  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    26  )
    27  
    28  // BatchEncoder is a maxwell format encoder implementation
    29  type BatchEncoder struct {
    30  	keyBuf      *bytes.Buffer
    31  	valueBuf    *bytes.Buffer
    32  	callbackBuf []func()
    33  	batchSize   int
    34  
    35  	config *common.Config
    36  }
    37  
    38  // EncodeCheckpointEvent implements the RowEventEncoder interface
    39  func (d *BatchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) {
    40  	// For maxwell now, there is no such a corresponding type to ResolvedEvent so far.
    41  	// Therefore the event is ignored.
    42  	return nil, nil
    43  }
    44  
    45  // AppendRowChangedEvent implements the RowEventEncoder interface
    46  func (d *BatchEncoder) AppendRowChangedEvent(
    47  	_ context.Context,
    48  	_ string,
    49  	e *model.RowChangedEvent,
    50  	callback func(),
    51  ) error {
    52  	_, valueMsg := rowChangeToMaxwellMsg(e, d.config.DeleteOnlyHandleKeyColumns)
    53  	value, err := valueMsg.encode()
    54  	if err != nil {
    55  		return errors.Trace(err)
    56  	}
    57  	d.valueBuf.Write(value)
    58  	d.batchSize++
    59  	if callback != nil {
    60  		d.callbackBuf = append(d.callbackBuf, callback)
    61  	}
    62  	return nil
    63  }
    64  
    65  // EncodeDDLEvent implements the RowEventEncoder interface
    66  // DDL message unresolved tso
    67  func (d *BatchEncoder) EncodeDDLEvent(e *model.DDLEvent) (*common.Message, error) {
    68  	keyMsg, valueMsg := ddlEventToMaxwellMsg(e)
    69  	key, err := keyMsg.Encode()
    70  	if err != nil {
    71  		return nil, errors.Trace(err)
    72  	}
    73  	value, err := valueMsg.encode()
    74  	if err != nil {
    75  		return nil, errors.Trace(err)
    76  	}
    77  
    78  	return common.NewDDLMsg(config.ProtocolMaxwell, key, value, e), nil
    79  }
    80  
    81  // Build implements the RowEventEncoder interface
    82  func (d *BatchEncoder) Build() []*common.Message {
    83  	if d.batchSize == 0 {
    84  		return nil
    85  	}
    86  
    87  	ret := common.NewMsg(config.ProtocolMaxwell,
    88  		d.keyBuf.Bytes(), d.valueBuf.Bytes(), 0, model.MessageTypeRow, nil, nil)
    89  	ret.SetRowsCount(d.batchSize)
    90  	if len(d.callbackBuf) != 0 && len(d.callbackBuf) == d.batchSize {
    91  		callbacks := d.callbackBuf
    92  		ret.Callback = func() {
    93  			for _, cb := range callbacks {
    94  				cb()
    95  			}
    96  		}
    97  		d.callbackBuf = make([]func(), 0)
    98  	}
    99  	d.reset()
   100  	return []*common.Message{ret}
   101  }
   102  
   103  // reset implements the RowEventEncoder interface
   104  func (d *BatchEncoder) reset() {
   105  	d.keyBuf.Reset()
   106  	d.valueBuf.Reset()
   107  	d.batchSize = 0
   108  	var versionByte [8]byte
   109  	binary.BigEndian.PutUint64(versionByte[:], codec.BatchVersion1)
   110  	d.keyBuf.Write(versionByte[:])
   111  }
   112  
   113  // newBatchEncoder creates a new maxwell BatchEncoder.
   114  func newBatchEncoder(config *common.Config) codec.RowEventEncoder {
   115  	batch := &BatchEncoder{
   116  		keyBuf:      &bytes.Buffer{},
   117  		valueBuf:    &bytes.Buffer{},
   118  		callbackBuf: make([]func(), 0),
   119  		config:      config,
   120  	}
   121  	batch.reset()
   122  	return batch
   123  }
   124  
   125  type batchEncoderBuilder struct {
   126  	config *common.Config
   127  }
   128  
   129  // NewBatchEncoderBuilder creates a maxwell batchEncoderBuilder.
   130  func NewBatchEncoderBuilder(config *common.Config) codec.RowEventEncoderBuilder {
   131  	return &batchEncoderBuilder{
   132  		config: config,
   133  	}
   134  }
   135  
   136  // Build a `maxwellBatchEncoder`
   137  func (b *batchEncoderBuilder) Build() codec.RowEventEncoder {
   138  	return newBatchEncoder(b.config)
   139  }
   140  
   141  // CleanMetrics do nothing
   142  func (b *batchEncoderBuilder) CleanMetrics() {}