vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package vreplication
    18  
    19  import (
    20  	"encoding/json"
    21  	"fmt"
    22  	"sort"
    23  	"strings"
    24  
    25  	"google.golang.org/protobuf/proto"
    26  
    27  	"vitess.io/vitess/go/bytes2"
    28  	"vitess.io/vitess/go/mysql"
    29  	"vitess.io/vitess/go/mysql/collations"
    30  	"vitess.io/vitess/go/sqltypes"
    31  	"vitess.io/vitess/go/vt/binlog/binlogplayer"
    32  	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
    33  	querypb "vitess.io/vitess/go/vt/proto/query"
    34  	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
    35  	"vitess.io/vitess/go/vt/sqlparser"
    36  	"vitess.io/vitess/go/vt/vterrors"
    37  	"vitess.io/vitess/go/vt/vtgate/evalengine"
    38  )
    39  
    40  // ReplicatorPlan is the execution plan for the replicator. It contains
    41  // plans for all the tables it's replicating. Every phase of vreplication
    42  // builds its own instance of the ReplicatorPlan. This is because the plan
    43  // depends on copyState, which changes on every iteration.
    44  // The table plans within ReplicatorPlan will not be fully populated because
    45  // all the information is not available initially.
    46  // For simplicity, the ReplicatorPlan is immutable.
    47  // Once we get the field info for a table from the stream response,
    48  // we'll have all the necessary info to build the final plan.
    49  // At that time, buildExecutionPlan is invoked, which will make a copy
    50  // of the TablePlan from ReplicatorPlan, and fill the rest
    51  // of the members, leaving the original plan unchanged.
    52  // The constructor is buildReplicatorPlan in table_plan_builder.go
    53  type ReplicatorPlan struct {
    54  	VStreamFilter *binlogdatapb.Filter
    55  	TargetTables  map[string]*TablePlan
    56  	TablePlans    map[string]*TablePlan
    57  	ColInfoMap    map[string][]*ColumnInfo
    58  	stats         *binlogplayer.Stats
    59  	Source        *binlogdatapb.BinlogSource
    60  }
    61  
    62  // buildExecution plan uses the field info as input and the partially built
    63  // TablePlan for that table to build a full plan.
    64  func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent) (*TablePlan, error) {
    65  	prelim := rp.TablePlans[fieldEvent.TableName]
    66  	if prelim == nil {
    67  		// Unreachable code.
    68  		return nil, fmt.Errorf("plan not found for %s", fieldEvent.TableName)
    69  	}
    70  	// If Insert is initialized, then it means that we knew the column
    71  	// names and have already built most of the plan.
    72  	if prelim.Insert != nil {
    73  		tplanv := *prelim
    74  		// We know that we sent only column names, but they may be backticked.
    75  		// If so, we have to strip them out to allow them to match the expected
    76  		// bind var names.
    77  		tplanv.Fields = make([]*querypb.Field, 0, len(fieldEvent.Fields))
    78  		for _, fld := range fieldEvent.Fields {
    79  			trimmed := proto.Clone(fld).(*querypb.Field)
    80  			trimmed.Name = strings.Trim(trimmed.Name, "`")
    81  			tplanv.Fields = append(tplanv.Fields, trimmed)
    82  		}
    83  		return &tplanv, nil
    84  	}
    85  	// select * construct was used. We need to use the field names.
    86  	tplan, err := rp.buildFromFields(prelim.TargetName, prelim.Lastpk, fieldEvent.Fields)
    87  	if err != nil {
    88  		return nil, err
    89  	}
    90  	tplan.Fields = fieldEvent.Fields
    91  	return tplan, nil
    92  }
    93  
    94  // buildFromFields builds a full TablePlan, but uses the field info as the
    95  // full column list. This happens when the query used was a 'select *', which
    96  // requires us to wait for the field info sent by the source.
    97  func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Result, fields []*querypb.Field) (*TablePlan, error) {
    98  	tpb := &tablePlanBuilder{
    99  		name:     sqlparser.NewIdentifierCS(tableName),
   100  		lastpk:   lastpk,
   101  		colInfos: rp.ColInfoMap[tableName],
   102  		stats:    rp.stats,
   103  		source:   rp.Source,
   104  	}
   105  	for _, field := range fields {
   106  		colName := sqlparser.NewIdentifierCI(field.Name)
   107  		isGenerated := false
   108  		for _, colInfo := range tpb.colInfos {
   109  			if !strings.EqualFold(colInfo.Name, field.Name) {
   110  				continue
   111  			}
   112  			if colInfo.IsGenerated {
   113  				isGenerated = true
   114  			}
   115  			break
   116  		}
   117  		if isGenerated {
   118  			continue
   119  		}
   120  		cexpr := &colExpr{
   121  			colName: colName,
   122  			colType: field.Type,
   123  			expr: &sqlparser.ColName{
   124  				Name: colName,
   125  			},
   126  			references: map[string]bool{
   127  				field.Name: true,
   128  			},
   129  		}
   130  		tpb.colExprs = append(tpb.colExprs, cexpr)
   131  	}
   132  	// The following actions are a subset of buildTablePlan.
   133  	if err := tpb.analyzePK(rp.ColInfoMap[tableName]); err != nil {
   134  		return nil, err
   135  	}
   136  	return tpb.generate(), nil
   137  }
   138  
   139  // MarshalJSON performs a custom JSON Marshalling.
   140  func (rp *ReplicatorPlan) MarshalJSON() ([]byte, error) {
   141  	var targets []string
   142  	for k := range rp.TargetTables {
   143  		targets = append(targets, k)
   144  	}
   145  	sort.Strings(targets)
   146  	v := struct {
   147  		VStreamFilter *binlogdatapb.Filter
   148  		TargetTables  []string
   149  		TablePlans    map[string]*TablePlan
   150  	}{
   151  		VStreamFilter: rp.VStreamFilter,
   152  		TargetTables:  targets,
   153  		TablePlans:    rp.TablePlans,
   154  	}
   155  	return json.Marshal(&v)
   156  }
   157  
   158  // TablePlan is the execution plan for a table within a replicator.
   159  // If the column names are not known at the time of plan building (like
   160  // select *), then only TargetName, SendRule and Lastpk are initialized.
   161  // When the stream returns the field info, those are used as column
   162  // names to build the final plan.
   163  // Lastpk comes from copyState. If it's set, then the generated plans
   164  // are significantly different because any events that fall beyond
   165  // Lastpk must be excluded.
   166  // If column names were known upfront, then all fields of TablePlan
   167  // are built except for Fields. This member is populated only after
   168  // the field info is received from the stream.
   169  // The ParsedQuery objects assume that a map of before and after values
   170  // will be built based on the streaming rows. Before image values will
   171  // be prefixed with a "b_", and after image values will be prefixed
   172  // with a "a_". The TablePlan structure is used during all the phases
   173  // of vreplication: catchup, copy, fastforward, or regular replication.
   174  type TablePlan struct {
   175  	// TargetName, SendRule will always be initialized.
   176  	TargetName string
   177  	SendRule   *binlogdatapb.Rule
   178  	// Lastpk will be initialized if it was specified, and
   179  	// will be used for building the final plan after field info
   180  	// is received.
   181  	Lastpk *sqltypes.Result
   182  	// BulkInsertFront, BulkInsertValues and BulkInsertOnDup are used
   183  	// by vcopier. These three parts are combined to build bulk insert
   184  	// statements. This is functionally equivalent to generating
   185  	// multiple statements using the "Insert" construct, but much more
   186  	// efficient for the copy phase.
   187  	BulkInsertFront  *sqlparser.ParsedQuery
   188  	BulkInsertValues *sqlparser.ParsedQuery
   189  	BulkInsertOnDup  *sqlparser.ParsedQuery
   190  	// Insert, Update and Delete are used by vplayer.
   191  	// If the plan is an insertIgnore type, then Insert
   192  	// and Update contain 'insert ignore' statements and
   193  	// Delete is nil.
   194  	Insert           *sqlparser.ParsedQuery
   195  	Update           *sqlparser.ParsedQuery
   196  	Delete           *sqlparser.ParsedQuery
   197  	Fields           []*querypb.Field
   198  	EnumValuesMap    map[string](map[string]string)
   199  	ConvertIntToEnum map[string]bool
   200  	// PKReferences is used to check if an event changed
   201  	// a primary key column (row move).
   202  	PKReferences            []string
   203  	Stats                   *binlogplayer.Stats
   204  	FieldsToSkip            map[string]bool
   205  	ConvertCharset          map[string](*binlogdatapb.CharsetConversion)
   206  	HasExtraSourcePkColumns bool
   207  }
   208  
   209  // MarshalJSON performs a custom JSON Marshalling.
   210  func (tp *TablePlan) MarshalJSON() ([]byte, error) {
   211  	v := struct {
   212  		TargetName   string
   213  		SendRule     string
   214  		InsertFront  *sqlparser.ParsedQuery `json:",omitempty"`
   215  		InsertValues *sqlparser.ParsedQuery `json:",omitempty"`
   216  		InsertOnDup  *sqlparser.ParsedQuery `json:",omitempty"`
   217  		Insert       *sqlparser.ParsedQuery `json:",omitempty"`
   218  		Update       *sqlparser.ParsedQuery `json:",omitempty"`
   219  		Delete       *sqlparser.ParsedQuery `json:",omitempty"`
   220  		PKReferences []string               `json:",omitempty"`
   221  	}{
   222  		TargetName:   tp.TargetName,
   223  		SendRule:     tp.SendRule.Match,
   224  		InsertFront:  tp.BulkInsertFront,
   225  		InsertValues: tp.BulkInsertValues,
   226  		InsertOnDup:  tp.BulkInsertOnDup,
   227  		Insert:       tp.Insert,
   228  		Update:       tp.Update,
   229  		Delete:       tp.Delete,
   230  		PKReferences: tp.PKReferences,
   231  	}
   232  	return json.Marshal(&v)
   233  }
   234  
   235  func (tp *TablePlan) applyBulkInsert(sqlbuffer *bytes2.Buffer, rows []*querypb.Row, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) {
   236  	sqlbuffer.Reset()
   237  	sqlbuffer.WriteString(tp.BulkInsertFront.Query)
   238  	sqlbuffer.WriteString(" values ")
   239  
   240  	for i, row := range rows {
   241  		if i > 0 {
   242  			sqlbuffer.WriteString(", ")
   243  		}
   244  		if err := tp.BulkInsertValues.AppendFromRow(sqlbuffer, tp.Fields, row, tp.FieldsToSkip); err != nil {
   245  			return nil, err
   246  		}
   247  	}
   248  	if tp.BulkInsertOnDup != nil {
   249  		sqlbuffer.WriteString(tp.BulkInsertOnDup.Query)
   250  	}
   251  	return executor(sqlbuffer.StringUnsafe())
   252  }
   253  
   254  // During the copy phase we run catchup and fastforward, which stream binlogs. While streaming we should only process
   255  // rows whose PK has already been copied. Ideally we should compare the PKs before applying the change and never send
   256  // such rows to the target mysql server. However reliably comparing primary keys in a manner compatible to MySQL will require a lot of
   257  // coding: consider composite PKs, character sets, collations ... So we send these rows to the mysql server which then does the comparison
   258  // in sql, through where clauses like "pk_val <= last_seen_pk".
   259  //
   260  // But this does generate a lot of unnecessary load of, effectively, no-ops since the where
   261  // clauses are always false. This can create a significant cpu load on the target for high qps servers resulting in a
   262  // much lower copy bandwidth (or provisioning more powerful servers).
   263  // isOutsidePKRange currently checks for rows with single primary keys which are currently comparable in Vitess:
   264  // (see NullsafeCompare() for types supported). It returns true if pk is not to be applied
   265  //
   266  // At this time we have decided to only perform this for Insert statements. Insert statements form a significant majority of
   267  // the generated noop load during catchup and are easier to test for. Update and Delete statements are very difficult to
   268  // unit test reliably and without flakiness with our current test framework. So as a pragmatic decision we support Insert
   269  // now and punt on the others.
   270  func (tp *TablePlan) isOutsidePKRange(bindvars map[string]*querypb.BindVariable, before, after bool, stmtType string) bool {
   271  	// added empty comments below, otherwise gofmt removes the spaces between the bitwise & and obfuscates this check!
   272  	if vreplicationExperimentalFlags /**/ & /**/ vreplicationExperimentalFlagOptimizeInserts == 0 {
   273  		return false
   274  	}
   275  	// Ensure there is one and only one value in lastpk and pkrefs.
   276  	if tp.Lastpk != nil && len(tp.Lastpk.Fields) == 1 && len(tp.Lastpk.Rows) == 1 && len(tp.Lastpk.Rows[0]) == 1 && len(tp.PKReferences) == 1 {
   277  		// check again that this is an insert
   278  		var bindvar *querypb.BindVariable
   279  		switch {
   280  		case !before && after:
   281  			bindvar = bindvars["a_"+tp.PKReferences[0]]
   282  		}
   283  		if bindvar == nil { //should never happen
   284  			return false
   285  		}
   286  
   287  		rowVal, _ := sqltypes.BindVariableToValue(bindvar)
   288  		// TODO(king-11) make collation aware
   289  		result, err := evalengine.NullsafeCompare(rowVal, tp.Lastpk.Rows[0][0], collations.Unknown)
   290  		// If rowVal is > last pk, transaction will be a noop, so don't apply this statement
   291  		if err == nil && result > 0 {
   292  			tp.Stats.NoopQueryCount.Add(stmtType, 1)
   293  			return true
   294  		}
   295  	}
   296  	return false
   297  }
   298  
   299  // bindFieldVal returns a bind variable based on given field and value.
   300  // Most values will just bind directly. But some values may need manipulation:
   301  // - text values with charset conversion
   302  // - enum values converted to text via Online DDL
   303  // - ...any other future possible values
   304  func (tp *TablePlan) bindFieldVal(field *querypb.Field, val *sqltypes.Value) (*querypb.BindVariable, error) {
   305  	if conversion, ok := tp.ConvertCharset[field.Name]; ok && !val.IsNull() {
   306  		// Non-null string value, for which we have a charset conversion instruction
   307  		valString := val.ToString()
   308  		fromEncoding, encodingOK := mysql.CharacterSetEncoding[conversion.FromCharset]
   309  		if !encodingOK {
   310  			return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", conversion.FromCharset, field.Name)
   311  		}
   312  		if fromEncoding != nil {
   313  			// As reminder, encoding can be nil for trivial charsets, like utf8 or ascii.
   314  			// encoding will be non-nil for charsets like latin1, gbk, etc.
   315  			var err error
   316  			valString, err = fromEncoding.NewDecoder().String(valString)
   317  			if err != nil {
   318  				return nil, err
   319  			}
   320  		}
   321  		return sqltypes.StringBindVariable(valString), nil
   322  	}
   323  	if tp.ConvertIntToEnum[field.Name] && !val.IsNull() {
   324  		// An integer converted to an enum. We must write the textual value of the int. i.e. 0 turns to '0'
   325  		return sqltypes.StringBindVariable(val.ToString()), nil
   326  	}
   327  	if enumValues, ok := tp.EnumValuesMap[field.Name]; ok && !val.IsNull() {
   328  		// The fact that this field has a EnumValuesMap entry, means we must
   329  		// use the enum's text value as opposed to the enum's numerical value.
   330  		// Once known use case is with Online DDL, when a column is converted from
   331  		// ENUM to a VARCHAR/TEXT.
   332  		enumValue, enumValueOK := enumValues[val.ToString()]
   333  		if !enumValueOK {
   334  			return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "Invalid enum value: %v for field %s", val, field.Name)
   335  		}
   336  		// get the enum text for this val
   337  		return sqltypes.StringBindVariable(enumValue), nil
   338  	}
   339  	if field.Type == querypb.Type_ENUM {
   340  		// This is an ENUM w/o a values map, which means that we are most likely using
   341  		// the index value -- what is stored and binlogged vs. the list of strings
   342  		// defined in the table schema -- and we must use an int bindvar or we'll have
   343  		// invalid/incorrect predicates like WHERE enumcol='2'.
   344  		// This will be the case when applying binlog events.
   345  		enumIndexVal := sqltypes.MakeTrusted(querypb.Type_UINT64, val.Raw())
   346  		if enumIndex, err := enumIndexVal.ToUint64(); err == nil {
   347  			return sqltypes.Uint64BindVariable(enumIndex), nil
   348  		}
   349  	}
   350  	return sqltypes.ValueBindVariable(*val), nil
   351  }
   352  
   353  func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) {
   354  	// MakeRowTrusted is needed here because Proto3ToResult is not convenient.
   355  	var before, after bool
   356  	bindvars := make(map[string]*querypb.BindVariable, len(tp.Fields))
   357  	if rowChange.Before != nil {
   358  		before = true
   359  		vals := sqltypes.MakeRowTrusted(tp.Fields, rowChange.Before)
   360  		for i, field := range tp.Fields {
   361  			bindVar, err := tp.bindFieldVal(field, &vals[i])
   362  			if err != nil {
   363  				return nil, err
   364  			}
   365  			bindvars["b_"+field.Name] = bindVar
   366  		}
   367  	}
   368  	if rowChange.After != nil {
   369  		after = true
   370  		vals := sqltypes.MakeRowTrusted(tp.Fields, rowChange.After)
   371  		for i, field := range tp.Fields {
   372  			bindVar, err := tp.bindFieldVal(field, &vals[i])
   373  			if err != nil {
   374  				return nil, err
   375  			}
   376  			bindvars["a_"+field.Name] = bindVar
   377  		}
   378  	}
   379  	switch {
   380  	case !before && after:
   381  		// only apply inserts for rows whose primary keys are within the range of rows already copied
   382  		if tp.isOutsidePKRange(bindvars, before, after, "insert") {
   383  			return nil, nil
   384  		}
   385  		return execParsedQuery(tp.Insert, bindvars, executor)
   386  	case before && !after:
   387  		if tp.Delete == nil {
   388  			return nil, nil
   389  		}
   390  		return execParsedQuery(tp.Delete, bindvars, executor)
   391  	case before && after:
   392  		if !tp.pkChanged(bindvars) && !tp.HasExtraSourcePkColumns {
   393  			return execParsedQuery(tp.Update, bindvars, executor)
   394  		}
   395  		if tp.Delete != nil {
   396  			if _, err := execParsedQuery(tp.Delete, bindvars, executor); err != nil {
   397  				return nil, err
   398  			}
   399  		}
   400  		if tp.isOutsidePKRange(bindvars, before, after, "insert") {
   401  			return nil, nil
   402  		}
   403  		return execParsedQuery(tp.Insert, bindvars, executor)
   404  	}
   405  	// Unreachable.
   406  	return nil, nil
   407  }
   408  
   409  func execParsedQuery(pq *sqlparser.ParsedQuery, bindvars map[string]*querypb.BindVariable, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) {
   410  	sql, err := pq.GenerateQuery(bindvars, nil)
   411  	if err != nil {
   412  		return nil, err
   413  	}
   414  	return executor(sql)
   415  }
   416  
   417  func (tp *TablePlan) pkChanged(bindvars map[string]*querypb.BindVariable) bool {
   418  	for _, pkref := range tp.PKReferences {
   419  		v1, _ := sqltypes.BindVariableToValue(bindvars["b_"+pkref])
   420  		v2, _ := sqltypes.BindVariableToValue(bindvars["a_"+pkref])
   421  		if !valsEqual(v1, v2) {
   422  			return true
   423  		}
   424  	}
   425  	return false
   426  }
   427  
   428  func valsEqual(v1, v2 sqltypes.Value) bool {
   429  	if v1.IsNull() && v2.IsNull() {
   430  		return true
   431  	}
   432  	// If any one of them is null, something has changed.
   433  	if v1.IsNull() || v2.IsNull() {
   434  		return false
   435  	}
   436  	// Compare content only if none are null.
   437  	return v1.ToString() == v2.ToString()
   438  }