github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/debezium/encoder.go (about)

     1  // Copyright 2024 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package debezium
    15  
    16  import (
    17  	"bytes"
    18  	"context"
    19  	"time"
    20  
    21  	"github.com/pingcap/tiflow/cdc/model"
    22  	"github.com/pingcap/tiflow/pkg/config"
    23  	"github.com/pingcap/tiflow/pkg/errors"
    24  	"github.com/pingcap/tiflow/pkg/sink/codec"
    25  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    26  )
    27  
    28  // BatchEncoder encodes message into Debezium format.
    29  type BatchEncoder struct {
    30  	messages []*common.Message
    31  
    32  	config *common.Config
    33  	codec  *dbzCodec
    34  }
    35  
    36  // EncodeCheckpointEvent implements the RowEventEncoder interface
    37  func (d *BatchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) {
    38  	// Currently ignored. Debezium MySQL Connector does not emit such event.
    39  	return nil, nil
    40  }
    41  
    42  // AppendRowChangedEvent implements the RowEventEncoder interface
    43  func (d *BatchEncoder) AppendRowChangedEvent(
    44  	_ context.Context,
    45  	_ string,
    46  	e *model.RowChangedEvent,
    47  	callback func(),
    48  ) error {
    49  	valueBuf := bytes.Buffer{}
    50  	err := d.codec.EncodeRowChangedEvent(e, &valueBuf)
    51  	if err != nil {
    52  		return errors.Trace(err)
    53  	}
    54  	// TODO: Use a streaming compression is better.
    55  	value, err := common.Compress(
    56  		d.config.ChangefeedID,
    57  		d.config.LargeMessageHandle.LargeMessageHandleCompression,
    58  		valueBuf.Bytes(),
    59  	)
    60  	if err != nil {
    61  		return errors.Trace(err)
    62  	}
    63  	m := &common.Message{
    64  		Key:      nil,
    65  		Value:    value,
    66  		Ts:       e.CommitTs,
    67  		Schema:   e.TableInfo.GetSchemaNamePtr(),
    68  		Table:    e.TableInfo.GetTableNamePtr(),
    69  		Type:     model.MessageTypeRow,
    70  		Protocol: config.ProtocolDebezium,
    71  		Callback: callback,
    72  	}
    73  	m.IncRowsCount()
    74  
    75  	d.messages = append(d.messages, m)
    76  	return nil
    77  }
    78  
    79  // EncodeDDLEvent implements the RowEventEncoder interface
    80  // DDL message unresolved tso
    81  func (d *BatchEncoder) EncodeDDLEvent(e *model.DDLEvent) (*common.Message, error) {
    82  	// Schema Change Events are currently not supported.
    83  	return nil, nil
    84  }
    85  
    86  // Build implements the RowEventEncoder interface
    87  func (d *BatchEncoder) Build() []*common.Message {
    88  	if len(d.messages) == 0 {
    89  		return nil
    90  	}
    91  
    92  	result := d.messages
    93  	d.messages = nil
    94  	return result
    95  }
    96  
    97  // newBatchEncoder creates a new Debezium BatchEncoder.
    98  func newBatchEncoder(c *common.Config, clusterID string) codec.RowEventEncoder {
    99  	batch := &BatchEncoder{
   100  		messages: nil,
   101  		config:   c,
   102  		codec: &dbzCodec{
   103  			config:    c,
   104  			clusterID: clusterID,
   105  			nowFunc:   time.Now,
   106  		},
   107  	}
   108  	return batch
   109  }
   110  
   111  type batchEncoderBuilder struct {
   112  	config    *common.Config
   113  	clusterID string
   114  }
   115  
   116  // NewBatchEncoderBuilder creates a Debezium batchEncoderBuilder.
   117  func NewBatchEncoderBuilder(config *common.Config, clusterID string) codec.RowEventEncoderBuilder {
   118  	return &batchEncoderBuilder{
   119  		config:    config,
   120  		clusterID: clusterID,
   121  	}
   122  }
   123  
   124  // Build a `BatchEncoder`
   125  func (b *batchEncoderBuilder) Build() codec.RowEventEncoder {
   126  	return newBatchEncoder(b.config, b.clusterID)
   127  }
   128  
   129  // CleanMetrics do nothing
   130  func (b *batchEncoderBuilder) CleanMetrics() {}