vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vreplication 18 19 import ( 20 "encoding/json" 21 "fmt" 22 "sort" 23 "strings" 24 25 "google.golang.org/protobuf/proto" 26 27 "vitess.io/vitess/go/bytes2" 28 "vitess.io/vitess/go/mysql" 29 "vitess.io/vitess/go/mysql/collations" 30 "vitess.io/vitess/go/sqltypes" 31 "vitess.io/vitess/go/vt/binlog/binlogplayer" 32 binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" 33 querypb "vitess.io/vitess/go/vt/proto/query" 34 vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" 35 "vitess.io/vitess/go/vt/sqlparser" 36 "vitess.io/vitess/go/vt/vterrors" 37 "vitess.io/vitess/go/vt/vtgate/evalengine" 38 ) 39 40 // ReplicatorPlan is the execution plan for the replicator. It contains 41 // plans for all the tables it's replicating. Every phase of vreplication 42 // builds its own instance of the ReplicatorPlan. This is because the plan 43 // depends on copyState, which changes on every iteration. 44 // The table plans within ReplicatorPlan will not be fully populated because 45 // all the information is not available initially. 46 // For simplicity, the ReplicatorPlan is immutable. 47 // Once we get the field info for a table from the stream response, 48 // we'll have all the necessary info to build the final plan. 49 // At that time, buildExecutionPlan is invoked, which will make a copy 50 // of the TablePlan from ReplicatorPlan, and fill the rest 51 // of the members, leaving the original plan unchanged. 52 // The constructor is buildReplicatorPlan in table_plan_builder.go 53 type ReplicatorPlan struct { 54 VStreamFilter *binlogdatapb.Filter 55 TargetTables map[string]*TablePlan 56 TablePlans map[string]*TablePlan 57 ColInfoMap map[string][]*ColumnInfo 58 stats *binlogplayer.Stats 59 Source *binlogdatapb.BinlogSource 60 } 61 62 // buildExecution plan uses the field info as input and the partially built 63 // TablePlan for that table to build a full plan. 64 func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent) (*TablePlan, error) { 65 prelim := rp.TablePlans[fieldEvent.TableName] 66 if prelim == nil { 67 // Unreachable code. 68 return nil, fmt.Errorf("plan not found for %s", fieldEvent.TableName) 69 } 70 // If Insert is initialized, then it means that we knew the column 71 // names and have already built most of the plan. 72 if prelim.Insert != nil { 73 tplanv := *prelim 74 // We know that we sent only column names, but they may be backticked. 75 // If so, we have to strip them out to allow them to match the expected 76 // bind var names. 77 tplanv.Fields = make([]*querypb.Field, 0, len(fieldEvent.Fields)) 78 for _, fld := range fieldEvent.Fields { 79 trimmed := proto.Clone(fld).(*querypb.Field) 80 trimmed.Name = strings.Trim(trimmed.Name, "`") 81 tplanv.Fields = append(tplanv.Fields, trimmed) 82 } 83 return &tplanv, nil 84 } 85 // select * construct was used. We need to use the field names. 86 tplan, err := rp.buildFromFields(prelim.TargetName, prelim.Lastpk, fieldEvent.Fields) 87 if err != nil { 88 return nil, err 89 } 90 tplan.Fields = fieldEvent.Fields 91 return tplan, nil 92 } 93 94 // buildFromFields builds a full TablePlan, but uses the field info as the 95 // full column list. This happens when the query used was a 'select *', which 96 // requires us to wait for the field info sent by the source. 97 func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Result, fields []*querypb.Field) (*TablePlan, error) { 98 tpb := &tablePlanBuilder{ 99 name: sqlparser.NewIdentifierCS(tableName), 100 lastpk: lastpk, 101 colInfos: rp.ColInfoMap[tableName], 102 stats: rp.stats, 103 source: rp.Source, 104 } 105 for _, field := range fields { 106 colName := sqlparser.NewIdentifierCI(field.Name) 107 isGenerated := false 108 for _, colInfo := range tpb.colInfos { 109 if !strings.EqualFold(colInfo.Name, field.Name) { 110 continue 111 } 112 if colInfo.IsGenerated { 113 isGenerated = true 114 } 115 break 116 } 117 if isGenerated { 118 continue 119 } 120 cexpr := &colExpr{ 121 colName: colName, 122 colType: field.Type, 123 expr: &sqlparser.ColName{ 124 Name: colName, 125 }, 126 references: map[string]bool{ 127 field.Name: true, 128 }, 129 } 130 tpb.colExprs = append(tpb.colExprs, cexpr) 131 } 132 // The following actions are a subset of buildTablePlan. 133 if err := tpb.analyzePK(rp.ColInfoMap[tableName]); err != nil { 134 return nil, err 135 } 136 return tpb.generate(), nil 137 } 138 139 // MarshalJSON performs a custom JSON Marshalling. 140 func (rp *ReplicatorPlan) MarshalJSON() ([]byte, error) { 141 var targets []string 142 for k := range rp.TargetTables { 143 targets = append(targets, k) 144 } 145 sort.Strings(targets) 146 v := struct { 147 VStreamFilter *binlogdatapb.Filter 148 TargetTables []string 149 TablePlans map[string]*TablePlan 150 }{ 151 VStreamFilter: rp.VStreamFilter, 152 TargetTables: targets, 153 TablePlans: rp.TablePlans, 154 } 155 return json.Marshal(&v) 156 } 157 158 // TablePlan is the execution plan for a table within a replicator. 159 // If the column names are not known at the time of plan building (like 160 // select *), then only TargetName, SendRule and Lastpk are initialized. 161 // When the stream returns the field info, those are used as column 162 // names to build the final plan. 163 // Lastpk comes from copyState. If it's set, then the generated plans 164 // are significantly different because any events that fall beyond 165 // Lastpk must be excluded. 166 // If column names were known upfront, then all fields of TablePlan 167 // are built except for Fields. This member is populated only after 168 // the field info is received from the stream. 169 // The ParsedQuery objects assume that a map of before and after values 170 // will be built based on the streaming rows. Before image values will 171 // be prefixed with a "b_", and after image values will be prefixed 172 // with a "a_". The TablePlan structure is used during all the phases 173 // of vreplication: catchup, copy, fastforward, or regular replication. 174 type TablePlan struct { 175 // TargetName, SendRule will always be initialized. 176 TargetName string 177 SendRule *binlogdatapb.Rule 178 // Lastpk will be initialized if it was specified, and 179 // will be used for building the final plan after field info 180 // is received. 181 Lastpk *sqltypes.Result 182 // BulkInsertFront, BulkInsertValues and BulkInsertOnDup are used 183 // by vcopier. These three parts are combined to build bulk insert 184 // statements. This is functionally equivalent to generating 185 // multiple statements using the "Insert" construct, but much more 186 // efficient for the copy phase. 187 BulkInsertFront *sqlparser.ParsedQuery 188 BulkInsertValues *sqlparser.ParsedQuery 189 BulkInsertOnDup *sqlparser.ParsedQuery 190 // Insert, Update and Delete are used by vplayer. 191 // If the plan is an insertIgnore type, then Insert 192 // and Update contain 'insert ignore' statements and 193 // Delete is nil. 194 Insert *sqlparser.ParsedQuery 195 Update *sqlparser.ParsedQuery 196 Delete *sqlparser.ParsedQuery 197 Fields []*querypb.Field 198 EnumValuesMap map[string](map[string]string) 199 ConvertIntToEnum map[string]bool 200 // PKReferences is used to check if an event changed 201 // a primary key column (row move). 202 PKReferences []string 203 Stats *binlogplayer.Stats 204 FieldsToSkip map[string]bool 205 ConvertCharset map[string](*binlogdatapb.CharsetConversion) 206 HasExtraSourcePkColumns bool 207 } 208 209 // MarshalJSON performs a custom JSON Marshalling. 210 func (tp *TablePlan) MarshalJSON() ([]byte, error) { 211 v := struct { 212 TargetName string 213 SendRule string 214 InsertFront *sqlparser.ParsedQuery `json:",omitempty"` 215 InsertValues *sqlparser.ParsedQuery `json:",omitempty"` 216 InsertOnDup *sqlparser.ParsedQuery `json:",omitempty"` 217 Insert *sqlparser.ParsedQuery `json:",omitempty"` 218 Update *sqlparser.ParsedQuery `json:",omitempty"` 219 Delete *sqlparser.ParsedQuery `json:",omitempty"` 220 PKReferences []string `json:",omitempty"` 221 }{ 222 TargetName: tp.TargetName, 223 SendRule: tp.SendRule.Match, 224 InsertFront: tp.BulkInsertFront, 225 InsertValues: tp.BulkInsertValues, 226 InsertOnDup: tp.BulkInsertOnDup, 227 Insert: tp.Insert, 228 Update: tp.Update, 229 Delete: tp.Delete, 230 PKReferences: tp.PKReferences, 231 } 232 return json.Marshal(&v) 233 } 234 235 func (tp *TablePlan) applyBulkInsert(sqlbuffer *bytes2.Buffer, rows []*querypb.Row, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) { 236 sqlbuffer.Reset() 237 sqlbuffer.WriteString(tp.BulkInsertFront.Query) 238 sqlbuffer.WriteString(" values ") 239 240 for i, row := range rows { 241 if i > 0 { 242 sqlbuffer.WriteString(", ") 243 } 244 if err := tp.BulkInsertValues.AppendFromRow(sqlbuffer, tp.Fields, row, tp.FieldsToSkip); err != nil { 245 return nil, err 246 } 247 } 248 if tp.BulkInsertOnDup != nil { 249 sqlbuffer.WriteString(tp.BulkInsertOnDup.Query) 250 } 251 return executor(sqlbuffer.StringUnsafe()) 252 } 253 254 // During the copy phase we run catchup and fastforward, which stream binlogs. While streaming we should only process 255 // rows whose PK has already been copied. Ideally we should compare the PKs before applying the change and never send 256 // such rows to the target mysql server. However reliably comparing primary keys in a manner compatible to MySQL will require a lot of 257 // coding: consider composite PKs, character sets, collations ... So we send these rows to the mysql server which then does the comparison 258 // in sql, through where clauses like "pk_val <= last_seen_pk". 259 // 260 // But this does generate a lot of unnecessary load of, effectively, no-ops since the where 261 // clauses are always false. This can create a significant cpu load on the target for high qps servers resulting in a 262 // much lower copy bandwidth (or provisioning more powerful servers). 263 // isOutsidePKRange currently checks for rows with single primary keys which are currently comparable in Vitess: 264 // (see NullsafeCompare() for types supported). It returns true if pk is not to be applied 265 // 266 // At this time we have decided to only perform this for Insert statements. Insert statements form a significant majority of 267 // the generated noop load during catchup and are easier to test for. Update and Delete statements are very difficult to 268 // unit test reliably and without flakiness with our current test framework. So as a pragmatic decision we support Insert 269 // now and punt on the others. 270 func (tp *TablePlan) isOutsidePKRange(bindvars map[string]*querypb.BindVariable, before, after bool, stmtType string) bool { 271 // added empty comments below, otherwise gofmt removes the spaces between the bitwise & and obfuscates this check! 272 if vreplicationExperimentalFlags /**/ & /**/ vreplicationExperimentalFlagOptimizeInserts == 0 { 273 return false 274 } 275 // Ensure there is one and only one value in lastpk and pkrefs. 276 if tp.Lastpk != nil && len(tp.Lastpk.Fields) == 1 && len(tp.Lastpk.Rows) == 1 && len(tp.Lastpk.Rows[0]) == 1 && len(tp.PKReferences) == 1 { 277 // check again that this is an insert 278 var bindvar *querypb.BindVariable 279 switch { 280 case !before && after: 281 bindvar = bindvars["a_"+tp.PKReferences[0]] 282 } 283 if bindvar == nil { //should never happen 284 return false 285 } 286 287 rowVal, _ := sqltypes.BindVariableToValue(bindvar) 288 // TODO(king-11) make collation aware 289 result, err := evalengine.NullsafeCompare(rowVal, tp.Lastpk.Rows[0][0], collations.Unknown) 290 // If rowVal is > last pk, transaction will be a noop, so don't apply this statement 291 if err == nil && result > 0 { 292 tp.Stats.NoopQueryCount.Add(stmtType, 1) 293 return true 294 } 295 } 296 return false 297 } 298 299 // bindFieldVal returns a bind variable based on given field and value. 300 // Most values will just bind directly. But some values may need manipulation: 301 // - text values with charset conversion 302 // - enum values converted to text via Online DDL 303 // - ...any other future possible values 304 func (tp *TablePlan) bindFieldVal(field *querypb.Field, val *sqltypes.Value) (*querypb.BindVariable, error) { 305 if conversion, ok := tp.ConvertCharset[field.Name]; ok && !val.IsNull() { 306 // Non-null string value, for which we have a charset conversion instruction 307 valString := val.ToString() 308 fromEncoding, encodingOK := mysql.CharacterSetEncoding[conversion.FromCharset] 309 if !encodingOK { 310 return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", conversion.FromCharset, field.Name) 311 } 312 if fromEncoding != nil { 313 // As reminder, encoding can be nil for trivial charsets, like utf8 or ascii. 314 // encoding will be non-nil for charsets like latin1, gbk, etc. 315 var err error 316 valString, err = fromEncoding.NewDecoder().String(valString) 317 if err != nil { 318 return nil, err 319 } 320 } 321 return sqltypes.StringBindVariable(valString), nil 322 } 323 if tp.ConvertIntToEnum[field.Name] && !val.IsNull() { 324 // An integer converted to an enum. We must write the textual value of the int. i.e. 0 turns to '0' 325 return sqltypes.StringBindVariable(val.ToString()), nil 326 } 327 if enumValues, ok := tp.EnumValuesMap[field.Name]; ok && !val.IsNull() { 328 // The fact that this field has a EnumValuesMap entry, means we must 329 // use the enum's text value as opposed to the enum's numerical value. 330 // Once known use case is with Online DDL, when a column is converted from 331 // ENUM to a VARCHAR/TEXT. 332 enumValue, enumValueOK := enumValues[val.ToString()] 333 if !enumValueOK { 334 return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Invalid enum value: %v for field %s", val, field.Name) 335 } 336 // get the enum text for this val 337 return sqltypes.StringBindVariable(enumValue), nil 338 } 339 if field.Type == querypb.Type_ENUM { 340 // This is an ENUM w/o a values map, which means that we are most likely using 341 // the index value -- what is stored and binlogged vs. the list of strings 342 // defined in the table schema -- and we must use an int bindvar or we'll have 343 // invalid/incorrect predicates like WHERE enumcol='2'. 344 // This will be the case when applying binlog events. 345 enumIndexVal := sqltypes.MakeTrusted(querypb.Type_UINT64, val.Raw()) 346 if enumIndex, err := enumIndexVal.ToUint64(); err == nil { 347 return sqltypes.Uint64BindVariable(enumIndex), nil 348 } 349 } 350 return sqltypes.ValueBindVariable(*val), nil 351 } 352 353 func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) { 354 // MakeRowTrusted is needed here because Proto3ToResult is not convenient. 355 var before, after bool 356 bindvars := make(map[string]*querypb.BindVariable, len(tp.Fields)) 357 if rowChange.Before != nil { 358 before = true 359 vals := sqltypes.MakeRowTrusted(tp.Fields, rowChange.Before) 360 for i, field := range tp.Fields { 361 bindVar, err := tp.bindFieldVal(field, &vals[i]) 362 if err != nil { 363 return nil, err 364 } 365 bindvars["b_"+field.Name] = bindVar 366 } 367 } 368 if rowChange.After != nil { 369 after = true 370 vals := sqltypes.MakeRowTrusted(tp.Fields, rowChange.After) 371 for i, field := range tp.Fields { 372 bindVar, err := tp.bindFieldVal(field, &vals[i]) 373 if err != nil { 374 return nil, err 375 } 376 bindvars["a_"+field.Name] = bindVar 377 } 378 } 379 switch { 380 case !before && after: 381 // only apply inserts for rows whose primary keys are within the range of rows already copied 382 if tp.isOutsidePKRange(bindvars, before, after, "insert") { 383 return nil, nil 384 } 385 return execParsedQuery(tp.Insert, bindvars, executor) 386 case before && !after: 387 if tp.Delete == nil { 388 return nil, nil 389 } 390 return execParsedQuery(tp.Delete, bindvars, executor) 391 case before && after: 392 if !tp.pkChanged(bindvars) && !tp.HasExtraSourcePkColumns { 393 return execParsedQuery(tp.Update, bindvars, executor) 394 } 395 if tp.Delete != nil { 396 if _, err := execParsedQuery(tp.Delete, bindvars, executor); err != nil { 397 return nil, err 398 } 399 } 400 if tp.isOutsidePKRange(bindvars, before, after, "insert") { 401 return nil, nil 402 } 403 return execParsedQuery(tp.Insert, bindvars, executor) 404 } 405 // Unreachable. 406 return nil, nil 407 } 408 409 func execParsedQuery(pq *sqlparser.ParsedQuery, bindvars map[string]*querypb.BindVariable, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) { 410 sql, err := pq.GenerateQuery(bindvars, nil) 411 if err != nil { 412 return nil, err 413 } 414 return executor(sql) 415 } 416 417 func (tp *TablePlan) pkChanged(bindvars map[string]*querypb.BindVariable) bool { 418 for _, pkref := range tp.PKReferences { 419 v1, _ := sqltypes.BindVariableToValue(bindvars["b_"+pkref]) 420 v2, _ := sqltypes.BindVariableToValue(bindvars["a_"+pkref]) 421 if !valsEqual(v1, v2) { 422 return true 423 } 424 } 425 return false 426 } 427 428 func valsEqual(v1, v2 sqltypes.Value) bool { 429 if v1.IsNull() && v2.IsNull() { 430 return true 431 } 432 // If any one of them is null, something has changed. 433 if v1.IsNull() || v2.IsNull() { 434 return false 435 } 436 // Compare content only if none are null. 437 return v1.ToString() == v2.ToString() 438 }