github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/canal/canal_json_txn_event_encoder.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package canal
    15  
    16  import (
    17  	"bytes"
    18  
    19  	"github.com/pingcap/errors"
    20  	"github.com/pingcap/log"
    21  	"github.com/pingcap/tiflow/cdc/model"
    22  	"github.com/pingcap/tiflow/pkg/config"
    23  	cerror "github.com/pingcap/tiflow/pkg/errors"
    24  	"github.com/pingcap/tiflow/pkg/sink/codec"
    25  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    26  	"go.uber.org/zap"
    27  )
    28  
    29  // JSONTxnEventEncoder encodes txn event in JSON format
    30  type JSONTxnEventEncoder struct {
    31  	builder *canalEntryBuilder
    32  
    33  	config *common.Config
    34  
    35  	// the symbol separating two lines
    36  	terminator []byte
    37  	valueBuf   *bytes.Buffer
    38  	batchSize  int
    39  	callback   func()
    40  
    41  	// Store some fields of the txn event.
    42  	txnCommitTs uint64
    43  	txnSchema   *string
    44  	txnTable    *string
    45  }
    46  
    47  // AppendTxnEvent appends a txn event to the encoder.
    48  func (j *JSONTxnEventEncoder) AppendTxnEvent(
    49  	txn *model.SingleTableTxn,
    50  	callback func(),
    51  ) error {
    52  	for _, row := range txn.Rows {
    53  		value, err := newJSONMessageForDML(j.builder, row, j.config, false, "")
    54  		if err != nil {
    55  			return errors.Trace(err)
    56  		}
    57  		length := len(value) + common.MaxRecordOverhead
    58  		// For single message that is longer than max-message-bytes, do not send it.
    59  		if length > j.config.MaxMessageBytes {
    60  			log.Warn("Single message is too large for canal-json",
    61  				zap.Int("maxMessageBytes", j.config.MaxMessageBytes),
    62  				zap.Int("length", length),
    63  				zap.Any("table", row.TableInfo.TableName))
    64  			return cerror.ErrMessageTooLarge.GenWithStackByArgs()
    65  		}
    66  		j.valueBuf.Write(value)
    67  		j.valueBuf.Write(j.terminator)
    68  		j.batchSize++
    69  	}
    70  	j.callback = callback
    71  	j.txnCommitTs = txn.CommitTs
    72  	j.txnSchema = txn.TableInfo.GetSchemaNamePtr()
    73  	j.txnTable = txn.TableInfo.GetTableNamePtr()
    74  	return nil
    75  }
    76  
    77  // Build builds a message from the encoder and resets the encoder.
    78  func (j *JSONTxnEventEncoder) Build() []*common.Message {
    79  	if j.batchSize == 0 {
    80  		return nil
    81  	}
    82  
    83  	ret := common.NewMsg(config.ProtocolCanalJSON, nil,
    84  		j.valueBuf.Bytes(), j.txnCommitTs, model.MessageTypeRow, j.txnSchema, j.txnTable)
    85  	ret.SetRowsCount(j.batchSize)
    86  	ret.Callback = j.callback
    87  	if j.valueBuf.Cap() > codec.MemBufShrinkThreshold {
    88  		j.valueBuf = &bytes.Buffer{}
    89  	} else {
    90  		j.valueBuf.Reset()
    91  	}
    92  	j.callback = nil
    93  	j.batchSize = 0
    94  	j.txnCommitTs = 0
    95  	j.txnSchema = nil
    96  	j.txnTable = nil
    97  
    98  	return []*common.Message{ret}
    99  }
   100  
   101  // newJSONTxnEventEncoder creates a new JSONTxnEventEncoder
   102  func newJSONTxnEventEncoder(config *common.Config) codec.TxnEventEncoder {
   103  	encoder := &JSONTxnEventEncoder{
   104  		builder:    newCanalEntryBuilder(config),
   105  		valueBuf:   &bytes.Buffer{},
   106  		terminator: []byte(config.Terminator),
   107  
   108  		config: config,
   109  	}
   110  	return encoder
   111  }
   112  
   113  type jsonTxnEventEncoderBuilder struct {
   114  	config *common.Config
   115  }
   116  
   117  // NewJSONTxnEventEncoderBuilder creates a jsonTxnEventEncoderBuilder.
   118  func NewJSONTxnEventEncoderBuilder(config *common.Config) codec.TxnEventEncoderBuilder {
   119  	return &jsonTxnEventEncoderBuilder{config: config}
   120  }
   121  
   122  // Build a `jsonTxnEventEncoderBuilder`
   123  func (b *jsonTxnEventEncoderBuilder) Build() codec.TxnEventEncoder {
   124  	return newJSONTxnEventEncoder(b.config)
   125  }