github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/maxwell/maxwell_message.go (about) 1 // Copyright 2022 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package maxwell 15 16 import ( 17 "encoding/json" 18 19 model2 "github.com/pingcap/tidb/pkg/parser/model" 20 "github.com/pingcap/tidb/pkg/parser/mysql" 21 "github.com/pingcap/tiflow/cdc/model" 22 cerror "github.com/pingcap/tiflow/pkg/errors" 23 "github.com/pingcap/tiflow/pkg/sink/codec/internal" 24 "github.com/tikv/client-go/v2/oracle" 25 ) 26 27 type maxwellMessage struct { 28 Database string `json:"database"` 29 Table string `json:"table"` 30 Type string `json:"type"` 31 Ts int64 `json:"ts"` 32 Xid int `json:"xid,omitempty"` 33 Xoffset int `json:"xoffset,omitempty"` 34 Position string `json:"position,omitempty"` 35 Gtid string `json:"gtid,omitempty"` 36 Data map[string]interface{} `json:"data,omitempty"` 37 Old map[string]interface{} `json:"old,omitempty"` 38 } 39 40 // Encode encodes the message to bytes 41 func (m *maxwellMessage) encode() ([]byte, error) { 42 data, err := json.Marshal(m) 43 return data, cerror.WrapError(cerror.ErrMaxwellEncodeFailed, err) 44 } 45 46 func rowChangeToMaxwellMsg(e *model.RowChangedEvent, onlyHandleKeyColumns bool) (*internal.MessageKey, *maxwellMessage) { 47 var partition *int64 48 if e.TableInfo.IsPartitionTable() { 49 partition = &e.PhysicalTableID 50 } 51 key := &internal.MessageKey{ 52 Ts: e.CommitTs, 53 Schema: e.TableInfo.GetSchemaName(), 54 Table: e.TableInfo.GetTableName(), 55 Partition: partition, 56 Type: model.MessageTypeRow, 57 } 58 value := &maxwellMessage{ 59 Ts: 0, 60 Database: e.TableInfo.GetSchemaName(), 61 Table: e.TableInfo.GetTableName(), 62 Data: make(map[string]interface{}), 63 Old: make(map[string]interface{}), 64 } 65 physicalTime := oracle.GetTimeFromTS(e.CommitTs) 66 value.Ts = physicalTime.Unix() 67 tableInfo := e.TableInfo 68 if e.IsDelete() { 69 value.Type = "delete" 70 for _, v := range e.PreColumns { 71 colFlag := tableInfo.ForceGetColumnFlagType(v.ColumnID) 72 if onlyHandleKeyColumns && !colFlag.IsHandleKey() { 73 continue 74 } 75 colInfo := tableInfo.ForceGetColumnInfo(v.ColumnID) 76 colName := tableInfo.ForceGetColumnName(v.ColumnID) 77 switch colInfo.GetType() { 78 case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: 79 if v.Value == nil { 80 value.Old[colName] = nil 81 } else if colFlag.IsBinary() { 82 value.Old[colName] = v.Value 83 } else { 84 value.Old[colName] = string(v.Value.([]byte)) 85 } 86 default: 87 value.Old[colName] = v.Value 88 } 89 } 90 } else { 91 for _, v := range e.Columns { 92 colFlag := tableInfo.ForceGetColumnFlagType(v.ColumnID) 93 colInfo := tableInfo.ForceGetColumnInfo(v.ColumnID) 94 colName := tableInfo.ForceGetColumnName(v.ColumnID) 95 switch colInfo.GetType() { 96 case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: 97 if v.Value == nil { 98 value.Data[colName] = nil 99 } else if colFlag.IsBinary() { 100 value.Data[colName] = v.Value 101 } else { 102 value.Data[colName] = string(v.Value.([]byte)) 103 } 104 default: 105 value.Data[colName] = v.Value 106 } 107 } 108 if e.PreColumns == nil { 109 value.Type = "insert" 110 } else { 111 value.Type = "update" 112 for _, v := range e.PreColumns { 113 colFlag := tableInfo.ForceGetColumnFlagType(v.ColumnID) 114 colInfo := tableInfo.ForceGetColumnInfo(v.ColumnID) 115 colName := tableInfo.ForceGetColumnName(v.ColumnID) 116 switch colInfo.GetType() { 117 case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: 118 if v.Value == nil { 119 if value.Data[colName] != nil { 120 value.Old[colName] = nil 121 } 122 } else if colFlag.IsBinary() { 123 if value.Data[colName] != v.Value { 124 value.Old[colName] = v.Value 125 } 126 } else { 127 if value.Data[colName] != string(v.Value.([]byte)) { 128 value.Old[colName] = string(v.Value.([]byte)) 129 } 130 } 131 default: 132 if value.Data[colName] != v.Value { 133 value.Old[colName] = v.Value 134 } 135 } 136 } 137 138 } 139 } 140 return key, value 141 } 142 143 // maxwellColumn represents a column in maxwell 144 type maxwellColumn struct { 145 Type string `json:"type"` 146 Name string `json:"name"` 147 // Do not mark the unique key temporarily 148 Signed bool `json:"signed,omitempty"` 149 ColumnLength int `json:"column-length,omitempty"` 150 Charset string `json:"charset,omitempty"` 151 } 152 153 // tableStruct represents a table structure includes some table info 154 type tableStruct struct { 155 Database string `json:"database"` 156 Charset string `json:"charset,omitempty"` 157 Table string `json:"table"` 158 Columns []*maxwellColumn `json:"columns"` 159 // Do not output whether it is a primary key temporarily 160 PrimaryKey []string `json:"primary-key"` 161 } 162 163 // ddlMaxwellMessage represents a DDL maxwell message 164 // Old for table old schema 165 // Def for table after ddl schema 166 type ddlMaxwellMessage struct { 167 Type string `json:"type"` 168 Database string `json:"database"` 169 Table string `json:"table"` 170 Old tableStruct `json:"old,omitempty"` 171 Def tableStruct `json:"def,omitempty"` 172 Ts uint64 `json:"ts"` 173 SQL string `json:"sql"` 174 Position string `json:"position,omitempty"` 175 } 176 177 // Encode encodes the message to bytes 178 func (m *ddlMaxwellMessage) encode() ([]byte, error) { 179 data, err := json.Marshal(m) 180 return data, cerror.WrapError(cerror.ErrMaxwellEncodeFailed, err) 181 } 182 183 func ddlEventToMaxwellMsg(e *model.DDLEvent) (*internal.MessageKey, *ddlMaxwellMessage) { 184 key := &internal.MessageKey{ 185 Ts: e.CommitTs, 186 Schema: e.TableInfo.TableName.Schema, 187 Table: e.TableInfo.TableName.Table, 188 Type: model.MessageTypeDDL, 189 } 190 value := &ddlMaxwellMessage{ 191 Ts: e.CommitTs, 192 Database: e.TableInfo.TableName.Schema, 193 Type: "table-create", 194 Table: e.TableInfo.TableName.Table, 195 Old: tableStruct{}, 196 Def: tableStruct{}, 197 SQL: e.Query, 198 } 199 200 value.Type = ddlToMaxwellType(e.Type) 201 202 if e.PreTableInfo != nil { 203 value.Old.Database = e.PreTableInfo.TableName.Schema 204 value.Old.Table = e.PreTableInfo.TableName.Table 205 for _, v := range e.PreTableInfo.TableInfo.Columns { 206 maxwellcolumntype, _ := columnToMaxwellType(v.FieldType.GetType()) 207 value.Old.Columns = append(value.Old.Columns, &maxwellColumn{ 208 Name: v.Name.O, 209 Type: maxwellcolumntype, 210 }) 211 } 212 } 213 214 value.Def.Database = e.TableInfo.TableName.Schema 215 value.Def.Table = e.TableInfo.TableName.Table 216 for _, v := range e.TableInfo.TableInfo.Columns { 217 maxwellcolumntype, err := columnToMaxwellType(v.FieldType.GetType()) 218 if err != nil { 219 value.Old.Columns = append(value.Old.Columns, &maxwellColumn{ 220 Name: v.Name.O, 221 Type: err.Error(), 222 }) 223 } 224 value.Def.Columns = append(value.Def.Columns, &maxwellColumn{ 225 Name: v.Name.O, 226 Type: maxwellcolumntype, 227 }) 228 } 229 return key, value 230 } 231 232 // ddl typecode from parser/model/ddl.go 233 func ddlToMaxwellType(ddlType model2.ActionType) string { 234 if ddlType >= model2.ActionAddColumn && ddlType <= model2.ActionDropTablePartition { 235 return "table-alter" 236 } 237 switch ddlType { 238 case model2.ActionCreateTable: 239 return "table-create" 240 case model2.ActionDropTable: 241 return "table-drop" 242 case 22, 23, 27, 28, 29, 33, 37, 38, 41, 42: 243 return "table-alter" 244 case model2.ActionCreateSchema: 245 return "database-create" 246 case model2.ActionDropSchema: 247 return "database-drop" 248 case model2.ActionModifySchemaCharsetAndCollate: 249 return "database-alter" 250 default: 251 return ddlType.String() 252 } 253 } 254 255 // Convert column type code to maxwell column type 256 func columnToMaxwellType(columnType byte) (string, error) { 257 switch columnType { 258 case mysql.TypeTiny, mysql.TypeShort, mysql.TypeLong, mysql.TypeInt24: 259 return "int", nil 260 case mysql.TypeLonglong: 261 return "bigint", nil 262 case mysql.TypeTinyBlob, mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeString, mysql.TypeVarchar: 263 return "string", nil 264 case mysql.TypeDate: 265 return "date", nil 266 case mysql.TypeTimestamp, mysql.TypeDatetime: 267 return "datetime", nil 268 case mysql.TypeDuration: 269 return "time", nil 270 case mysql.TypeYear: 271 return "year", nil 272 case mysql.TypeEnum: 273 return "enum", nil 274 case mysql.TypeSet: 275 return "set", nil 276 case mysql.TypeBit: 277 return "bit", nil 278 case mysql.TypeJSON: 279 return "json", nil 280 case mysql.TypeFloat, mysql.TypeDouble: 281 return "float", nil 282 case mysql.TypeNewDecimal: 283 return "decimal", nil 284 default: 285 return "", cerror.ErrMaxwellInvalidData.GenWithStack("unsupported column type - %v", columnType) 286 } 287 }