github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/simple/marshaller.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package simple
    15  
    16  import (
    17  	_ "embed"
    18  	"encoding/json"
    19  
    20  	"github.com/linkedin/goavro/v2"
    21  	"github.com/pingcap/tiflow/cdc/model"
    22  	"github.com/pingcap/tiflow/pkg/errors"
    23  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    24  )
    25  
    26  //go:embed message.json
    27  var avroSchemaBytes []byte
    28  
    29  type marshaller interface {
    30  	// MarshalCheckpoint marshals the checkpoint ts into bytes.
    31  	MarshalCheckpoint(ts uint64) ([]byte, error)
    32  
    33  	// MarshalDDLEvent marshals the DDL event into bytes.
    34  	MarshalDDLEvent(event *model.DDLEvent) ([]byte, error)
    35  
    36  	// MarshalRowChangedEvent marshals the row changed event into bytes.
    37  	MarshalRowChangedEvent(event *model.RowChangedEvent,
    38  		handleKeyOnly bool, claimCheckFileName string) ([]byte, error)
    39  
    40  	// Unmarshal the bytes into the given value.
    41  	Unmarshal(data []byte, v any) error
    42  }
    43  
    44  func newMarshaller(config *common.Config) (marshaller, error) {
    45  	var (
    46  		result marshaller
    47  		err    error
    48  	)
    49  	switch config.EncodingFormat {
    50  	case common.EncodingFormatJSON:
    51  		result = newJSONMarshaller(config)
    52  	case common.EncodingFormatAvro:
    53  		result, err = newAvroMarshaller(config, string(avroSchemaBytes))
    54  	}
    55  	return result, errors.Trace(err)
    56  }
    57  
    58  type jsonMarshaller struct {
    59  	config *common.Config
    60  }
    61  
    62  func newJSONMarshaller(config *common.Config) *jsonMarshaller {
    63  	return &jsonMarshaller{
    64  		config: config,
    65  	}
    66  }
    67  
    68  // MarshalCheckpoint implement the marshaller interface
    69  func (m *jsonMarshaller) MarshalCheckpoint(ts uint64) ([]byte, error) {
    70  	msg := newResolvedMessage(ts)
    71  	result, err := json.Marshal(msg)
    72  	return result, errors.WrapError(errors.ErrEncodeFailed, err)
    73  }
    74  
    75  // MarshalDDLEvent implement the marshaller interface
    76  func (m *jsonMarshaller) MarshalDDLEvent(event *model.DDLEvent) ([]byte, error) {
    77  	var msg *message
    78  	if event.IsBootstrap {
    79  		msg = newBootstrapMessage(event.TableInfo)
    80  	} else {
    81  		msg = newDDLMessage(event)
    82  	}
    83  	value, err := json.Marshal(msg)
    84  	return value, errors.WrapError(errors.ErrEncodeFailed, err)
    85  }
    86  
    87  // MarshalRowChangedEvent implement the marshaller interface
    88  func (m *jsonMarshaller) MarshalRowChangedEvent(
    89  	event *model.RowChangedEvent,
    90  	handleKeyOnly bool, claimCheckFileName string,
    91  ) ([]byte, error) {
    92  	msg := m.newDMLMessage(event, handleKeyOnly, claimCheckFileName)
    93  	value, err := json.Marshal(msg)
    94  	return value, errors.WrapError(errors.ErrEncodeFailed, err)
    95  }
    96  
    97  // Unmarshal implement the marshaller interface
    98  func (m *jsonMarshaller) Unmarshal(data []byte, v any) error {
    99  	return json.Unmarshal(data, v)
   100  }
   101  
   102  type avroMarshaller struct {
   103  	codec  *goavro.Codec
   104  	config *common.Config
   105  }
   106  
   107  func newAvroMarshaller(config *common.Config, schema string) (*avroMarshaller, error) {
   108  	codec, err := goavro.NewCodec(schema)
   109  	return &avroMarshaller{
   110  		codec:  codec,
   111  		config: config,
   112  	}, errors.Trace(err)
   113  }
   114  
   115  // MarshalCheckpoint implement the marshaller interface
   116  func (m *avroMarshaller) MarshalCheckpoint(ts uint64) ([]byte, error) {
   117  	msg := newResolvedMessageMap(ts)
   118  	result, err := m.codec.BinaryFromNative(nil, msg)
   119  	return result, errors.WrapError(errors.ErrEncodeFailed, err)
   120  }
   121  
   122  // MarshalDDLEvent implement the marshaller interface
   123  func (m *avroMarshaller) MarshalDDLEvent(event *model.DDLEvent) ([]byte, error) {
   124  	var msg map[string]interface{}
   125  	if event.IsBootstrap {
   126  		msg = newBootstrapMessageMap(event.TableInfo)
   127  	} else {
   128  		msg = newDDLMessageMap(event)
   129  	}
   130  	value, err := m.codec.BinaryFromNative(nil, msg)
   131  	return value, errors.WrapError(errors.ErrEncodeFailed, err)
   132  }
   133  
   134  // MarshalRowChangedEvent implement the marshaller interface
   135  func (m *avroMarshaller) MarshalRowChangedEvent(
   136  	event *model.RowChangedEvent,
   137  	handleKeyOnly bool, claimCheckFileName string,
   138  ) ([]byte, error) {
   139  	msg := m.newDMLMessageMap(event, handleKeyOnly, claimCheckFileName)
   140  	value, err := m.codec.BinaryFromNative(nil, msg)
   141  	recycleMap(msg)
   142  	return value, errors.WrapError(errors.ErrEncodeFailed, err)
   143  }
   144  
   145  // Unmarshal implement the marshaller interface
   146  func (m *avroMarshaller) Unmarshal(data []byte, v any) error {
   147  	native, _, err := m.codec.NativeFromBinary(data)
   148  	newMessageFromAvroNative(native, v.(*message))
   149  	return errors.Trace(err)
   150  }