github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/maxwell/maxwell_message.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package maxwell
    15  
    16  import (
    17  	"encoding/json"
    18  
    19  	model2 "github.com/pingcap/tidb/pkg/parser/model"
    20  	"github.com/pingcap/tidb/pkg/parser/mysql"
    21  	"github.com/pingcap/tiflow/cdc/model"
    22  	cerror "github.com/pingcap/tiflow/pkg/errors"
    23  	"github.com/pingcap/tiflow/pkg/sink/codec/internal"
    24  	"github.com/tikv/client-go/v2/oracle"
    25  )
    26  
    27  type maxwellMessage struct {
    28  	Database string                 `json:"database"`
    29  	Table    string                 `json:"table"`
    30  	Type     string                 `json:"type"`
    31  	Ts       int64                  `json:"ts"`
    32  	Xid      int                    `json:"xid,omitempty"`
    33  	Xoffset  int                    `json:"xoffset,omitempty"`
    34  	Position string                 `json:"position,omitempty"`
    35  	Gtid     string                 `json:"gtid,omitempty"`
    36  	Data     map[string]interface{} `json:"data,omitempty"`
    37  	Old      map[string]interface{} `json:"old,omitempty"`
    38  }
    39  
    40  // Encode encodes the message to bytes
    41  func (m *maxwellMessage) encode() ([]byte, error) {
    42  	data, err := json.Marshal(m)
    43  	return data, cerror.WrapError(cerror.ErrMaxwellEncodeFailed, err)
    44  }
    45  
    46  func rowChangeToMaxwellMsg(e *model.RowChangedEvent, onlyHandleKeyColumns bool) (*internal.MessageKey, *maxwellMessage) {
    47  	var partition *int64
    48  	if e.TableInfo.IsPartitionTable() {
    49  		partition = &e.PhysicalTableID
    50  	}
    51  	key := &internal.MessageKey{
    52  		Ts:        e.CommitTs,
    53  		Schema:    e.TableInfo.GetSchemaName(),
    54  		Table:     e.TableInfo.GetTableName(),
    55  		Partition: partition,
    56  		Type:      model.MessageTypeRow,
    57  	}
    58  	value := &maxwellMessage{
    59  		Ts:       0,
    60  		Database: e.TableInfo.GetSchemaName(),
    61  		Table:    e.TableInfo.GetTableName(),
    62  		Data:     make(map[string]interface{}),
    63  		Old:      make(map[string]interface{}),
    64  	}
    65  	physicalTime := oracle.GetTimeFromTS(e.CommitTs)
    66  	value.Ts = physicalTime.Unix()
    67  	tableInfo := e.TableInfo
    68  	if e.IsDelete() {
    69  		value.Type = "delete"
    70  		for _, v := range e.PreColumns {
    71  			colFlag := tableInfo.ForceGetColumnFlagType(v.ColumnID)
    72  			if onlyHandleKeyColumns && !colFlag.IsHandleKey() {
    73  				continue
    74  			}
    75  			colInfo := tableInfo.ForceGetColumnInfo(v.ColumnID)
    76  			colName := tableInfo.ForceGetColumnName(v.ColumnID)
    77  			switch colInfo.GetType() {
    78  			case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob:
    79  				if v.Value == nil {
    80  					value.Old[colName] = nil
    81  				} else if colFlag.IsBinary() {
    82  					value.Old[colName] = v.Value
    83  				} else {
    84  					value.Old[colName] = string(v.Value.([]byte))
    85  				}
    86  			default:
    87  				value.Old[colName] = v.Value
    88  			}
    89  		}
    90  	} else {
    91  		for _, v := range e.Columns {
    92  			colFlag := tableInfo.ForceGetColumnFlagType(v.ColumnID)
    93  			colInfo := tableInfo.ForceGetColumnInfo(v.ColumnID)
    94  			colName := tableInfo.ForceGetColumnName(v.ColumnID)
    95  			switch colInfo.GetType() {
    96  			case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob:
    97  				if v.Value == nil {
    98  					value.Data[colName] = nil
    99  				} else if colFlag.IsBinary() {
   100  					value.Data[colName] = v.Value
   101  				} else {
   102  					value.Data[colName] = string(v.Value.([]byte))
   103  				}
   104  			default:
   105  				value.Data[colName] = v.Value
   106  			}
   107  		}
   108  		if e.PreColumns == nil {
   109  			value.Type = "insert"
   110  		} else {
   111  			value.Type = "update"
   112  			for _, v := range e.PreColumns {
   113  				colFlag := tableInfo.ForceGetColumnFlagType(v.ColumnID)
   114  				colInfo := tableInfo.ForceGetColumnInfo(v.ColumnID)
   115  				colName := tableInfo.ForceGetColumnName(v.ColumnID)
   116  				switch colInfo.GetType() {
   117  				case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob:
   118  					if v.Value == nil {
   119  						if value.Data[colName] != nil {
   120  							value.Old[colName] = nil
   121  						}
   122  					} else if colFlag.IsBinary() {
   123  						if value.Data[colName] != v.Value {
   124  							value.Old[colName] = v.Value
   125  						}
   126  					} else {
   127  						if value.Data[colName] != string(v.Value.([]byte)) {
   128  							value.Old[colName] = string(v.Value.([]byte))
   129  						}
   130  					}
   131  				default:
   132  					if value.Data[colName] != v.Value {
   133  						value.Old[colName] = v.Value
   134  					}
   135  				}
   136  			}
   137  
   138  		}
   139  	}
   140  	return key, value
   141  }
   142  
   143  // maxwellColumn represents a column in maxwell
   144  type maxwellColumn struct {
   145  	Type string `json:"type"`
   146  	Name string `json:"name"`
   147  	// Do not mark the unique key temporarily
   148  	Signed       bool   `json:"signed,omitempty"`
   149  	ColumnLength int    `json:"column-length,omitempty"`
   150  	Charset      string `json:"charset,omitempty"`
   151  }
   152  
   153  // tableStruct represents a table structure includes some table info
   154  type tableStruct struct {
   155  	Database string           `json:"database"`
   156  	Charset  string           `json:"charset,omitempty"`
   157  	Table    string           `json:"table"`
   158  	Columns  []*maxwellColumn `json:"columns"`
   159  	// Do not output whether it is a primary key temporarily
   160  	PrimaryKey []string `json:"primary-key"`
   161  }
   162  
   163  // ddlMaxwellMessage represents a DDL maxwell message
   164  // Old for table old schema
   165  // Def for table after ddl schema
   166  type ddlMaxwellMessage struct {
   167  	Type     string      `json:"type"`
   168  	Database string      `json:"database"`
   169  	Table    string      `json:"table"`
   170  	Old      tableStruct `json:"old,omitempty"`
   171  	Def      tableStruct `json:"def,omitempty"`
   172  	Ts       uint64      `json:"ts"`
   173  	SQL      string      `json:"sql"`
   174  	Position string      `json:"position,omitempty"`
   175  }
   176  
   177  // Encode encodes the message to bytes
   178  func (m *ddlMaxwellMessage) encode() ([]byte, error) {
   179  	data, err := json.Marshal(m)
   180  	return data, cerror.WrapError(cerror.ErrMaxwellEncodeFailed, err)
   181  }
   182  
   183  func ddlEventToMaxwellMsg(e *model.DDLEvent) (*internal.MessageKey, *ddlMaxwellMessage) {
   184  	key := &internal.MessageKey{
   185  		Ts:     e.CommitTs,
   186  		Schema: e.TableInfo.TableName.Schema,
   187  		Table:  e.TableInfo.TableName.Table,
   188  		Type:   model.MessageTypeDDL,
   189  	}
   190  	value := &ddlMaxwellMessage{
   191  		Ts:       e.CommitTs,
   192  		Database: e.TableInfo.TableName.Schema,
   193  		Type:     "table-create",
   194  		Table:    e.TableInfo.TableName.Table,
   195  		Old:      tableStruct{},
   196  		Def:      tableStruct{},
   197  		SQL:      e.Query,
   198  	}
   199  
   200  	value.Type = ddlToMaxwellType(e.Type)
   201  
   202  	if e.PreTableInfo != nil {
   203  		value.Old.Database = e.PreTableInfo.TableName.Schema
   204  		value.Old.Table = e.PreTableInfo.TableName.Table
   205  		for _, v := range e.PreTableInfo.TableInfo.Columns {
   206  			maxwellcolumntype, _ := columnToMaxwellType(v.FieldType.GetType())
   207  			value.Old.Columns = append(value.Old.Columns, &maxwellColumn{
   208  				Name: v.Name.O,
   209  				Type: maxwellcolumntype,
   210  			})
   211  		}
   212  	}
   213  
   214  	value.Def.Database = e.TableInfo.TableName.Schema
   215  	value.Def.Table = e.TableInfo.TableName.Table
   216  	for _, v := range e.TableInfo.TableInfo.Columns {
   217  		maxwellcolumntype, err := columnToMaxwellType(v.FieldType.GetType())
   218  		if err != nil {
   219  			value.Old.Columns = append(value.Old.Columns, &maxwellColumn{
   220  				Name: v.Name.O,
   221  				Type: err.Error(),
   222  			})
   223  		}
   224  		value.Def.Columns = append(value.Def.Columns, &maxwellColumn{
   225  			Name: v.Name.O,
   226  			Type: maxwellcolumntype,
   227  		})
   228  	}
   229  	return key, value
   230  }
   231  
   232  // ddl typecode from parser/model/ddl.go
   233  func ddlToMaxwellType(ddlType model2.ActionType) string {
   234  	if ddlType >= model2.ActionAddColumn && ddlType <= model2.ActionDropTablePartition {
   235  		return "table-alter"
   236  	}
   237  	switch ddlType {
   238  	case model2.ActionCreateTable:
   239  		return "table-create"
   240  	case model2.ActionDropTable:
   241  		return "table-drop"
   242  	case 22, 23, 27, 28, 29, 33, 37, 38, 41, 42:
   243  		return "table-alter"
   244  	case model2.ActionCreateSchema:
   245  		return "database-create"
   246  	case model2.ActionDropSchema:
   247  		return "database-drop"
   248  	case model2.ActionModifySchemaCharsetAndCollate:
   249  		return "database-alter"
   250  	default:
   251  		return ddlType.String()
   252  	}
   253  }
   254  
   255  // Convert column type code to maxwell column type
   256  func columnToMaxwellType(columnType byte) (string, error) {
   257  	switch columnType {
   258  	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeLong, mysql.TypeInt24:
   259  		return "int", nil
   260  	case mysql.TypeLonglong:
   261  		return "bigint", nil
   262  	case mysql.TypeTinyBlob, mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeString, mysql.TypeVarchar:
   263  		return "string", nil
   264  	case mysql.TypeDate:
   265  		return "date", nil
   266  	case mysql.TypeTimestamp, mysql.TypeDatetime:
   267  		return "datetime", nil
   268  	case mysql.TypeDuration:
   269  		return "time", nil
   270  	case mysql.TypeYear:
   271  		return "year", nil
   272  	case mysql.TypeEnum:
   273  		return "enum", nil
   274  	case mysql.TypeSet:
   275  		return "set", nil
   276  	case mysql.TypeBit:
   277  		return "bit", nil
   278  	case mysql.TypeJSON:
   279  		return "json", nil
   280  	case mysql.TypeFloat, mysql.TypeDouble:
   281  		return "float", nil
   282  	case mysql.TypeNewDecimal:
   283  		return "decimal", nil
   284  	default:
   285  		return "", cerror.ErrMaxwellInvalidData.GenWithStack("unsupported column type - %v", columnType)
   286  	}
   287  }