vitess.io/vitess@v0.16.2/go/vt/vtgate/vindexes/consistent_lookup.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package vindexes
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	"fmt"
    24  
    25  	"vitess.io/vitess/go/mysql"
    26  	"vitess.io/vitess/go/mysql/collations"
    27  	"vitess.io/vitess/go/vt/vtgate/evalengine"
    28  
    29  	"vitess.io/vitess/go/sqltypes"
    30  	"vitess.io/vitess/go/vt/key"
    31  	querypb "vitess.io/vitess/go/vt/proto/query"
    32  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    33  	"vitess.io/vitess/go/vt/proto/vtgate"
    34  	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
    35  	"vitess.io/vitess/go/vt/sqlparser"
    36  )
    37  
    38  var (
    39  	_ SingleColumn   = (*ConsistentLookupUnique)(nil)
    40  	_ Lookup         = (*ConsistentLookupUnique)(nil)
    41  	_ WantOwnerInfo  = (*ConsistentLookupUnique)(nil)
    42  	_ LookupPlanable = (*ConsistentLookupUnique)(nil)
    43  	_ SingleColumn   = (*ConsistentLookup)(nil)
    44  	_ Lookup         = (*ConsistentLookup)(nil)
    45  	_ WantOwnerInfo  = (*ConsistentLookup)(nil)
    46  	_ LookupPlanable = (*ConsistentLookup)(nil)
    47  )
    48  
    49  func init() {
    50  	Register("consistent_lookup", NewConsistentLookup)
    51  	Register("consistent_lookup_unique", NewConsistentLookupUnique)
    52  }
    53  
    54  // ConsistentLookup is a non-unique lookup vindex that can stay
    55  // consistent with respect to its owner table.
    56  type ConsistentLookup struct {
    57  	*clCommon
    58  }
    59  
    60  // NewConsistentLookup creates a ConsistentLookup vindex.
    61  // The supplied map has the following required fields:
    62  //
    63  //	table: name of the backing table. It can be qualified by the keyspace.
    64  //	from: list of columns in the table that have the 'from' values of the lookup vindex.
    65  //	to: The 'to' column name of the table.
    66  func NewConsistentLookup(name string, m map[string]string) (Vindex, error) {
    67  	clc, err := newCLCommon(name, m)
    68  	if err != nil {
    69  		return nil, err
    70  	}
    71  	return &ConsistentLookup{clCommon: clc}, nil
    72  }
    73  
    74  // Cost returns the cost of this vindex as 20.
    75  func (lu *ConsistentLookup) Cost() int {
    76  	return 20
    77  }
    78  
    79  // IsUnique returns false since the Vindex is non unique.
    80  func (lu *ConsistentLookup) IsUnique() bool {
    81  	return false
    82  }
    83  
    84  // NeedsVCursor satisfies the Vindex interface.
    85  func (lu *ConsistentLookup) NeedsVCursor() bool {
    86  	return true
    87  }
    88  
    89  // Map can map ids to key.Destination objects.
    90  func (lu *ConsistentLookup) Map(ctx context.Context, vcursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) {
    91  	out := make([]key.Destination, 0, len(ids))
    92  	if lu.writeOnly {
    93  		for range ids {
    94  			out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}})
    95  		}
    96  		return out, nil
    97  	}
    98  
    99  	// if ignore_nulls is set and the query is about single null value, then fallback to all shards
   100  	if len(ids) == 1 && ids[0].IsNull() && lu.lkp.IgnoreNulls {
   101  		for range ids {
   102  			out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}})
   103  		}
   104  		return out, nil
   105  	}
   106  
   107  	results, err := lu.lkp.Lookup(ctx, vcursor, ids, vcursor.LookupRowLockShardSession())
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	return lu.MapResult(ids, results)
   112  }
   113  
   114  // MapResult implements the LookupPlanable interface
   115  func (lu *ConsistentLookup) MapResult(ids []sqltypes.Value, results []*sqltypes.Result) ([]key.Destination, error) {
   116  	out := make([]key.Destination, 0, len(ids))
   117  	if lu.writeOnly {
   118  		for range ids {
   119  			out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}})
   120  		}
   121  		return out, nil
   122  	}
   123  	for _, result := range results {
   124  		if len(result.Rows) == 0 {
   125  			out = append(out, key.DestinationNone{})
   126  			continue
   127  		}
   128  		ksids := make([][]byte, 0, len(result.Rows))
   129  		for _, row := range result.Rows {
   130  			rowBytes, err := row[0].ToBytes()
   131  			if err != nil {
   132  				return nil, err
   133  			}
   134  			ksids = append(ksids, rowBytes)
   135  		}
   136  		out = append(out, key.DestinationKeyspaceIDs(ksids))
   137  	}
   138  	return out, nil
   139  }
   140  
   141  // Query implements the LookupPlanable interface
   142  func (lu *ConsistentLookup) Query() (selQuery string, arguments []string) {
   143  	return lu.lkp.query()
   144  }
   145  
   146  // AllowBatch implements the LookupPlanable interface
   147  func (lu *ConsistentLookup) AllowBatch() bool {
   148  	return lu.lkp.BatchLookup
   149  }
   150  
   151  func (lu *ConsistentLookup) AutoCommitEnabled() bool {
   152  	return lu.lkp.Autocommit
   153  }
   154  
   155  //====================================================================
   156  
   157  // ConsistentLookupUnique defines a vindex that uses a lookup table.
   158  // The table is expected to define the id column as unique. It's
   159  // Unique and a Lookup.
   160  type ConsistentLookupUnique struct {
   161  	*clCommon
   162  }
   163  
   164  // NewConsistentLookupUnique creates a ConsistentLookupUnique vindex.
   165  // The supplied map has the following required fields:
   166  //
   167  //	table: name of the backing table. It can be qualified by the keyspace.
   168  //	from: list of columns in the table that have the 'from' values of the lookup vindex.
   169  //	to: The 'to' column name of the table.
   170  func NewConsistentLookupUnique(name string, m map[string]string) (Vindex, error) {
   171  	clc, err := newCLCommon(name, m)
   172  	if err != nil {
   173  		return nil, err
   174  	}
   175  	return &ConsistentLookupUnique{clCommon: clc}, nil
   176  }
   177  
   178  // Cost returns the cost of this vindex as 10.
   179  func (lu *ConsistentLookupUnique) Cost() int {
   180  	return 10
   181  }
   182  
   183  // IsUnique returns true since the Vindex is unique.
   184  func (lu *ConsistentLookupUnique) IsUnique() bool {
   185  	return true
   186  }
   187  
   188  // NeedsVCursor satisfies the Vindex interface.
   189  func (lu *ConsistentLookupUnique) NeedsVCursor() bool {
   190  	return true
   191  }
   192  
   193  // Map can map ids to key.Destination objects.
   194  func (lu *ConsistentLookupUnique) Map(ctx context.Context, vcursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) {
   195  	out := make([]key.Destination, 0, len(ids))
   196  	if lu.writeOnly {
   197  		for range ids {
   198  			out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}})
   199  		}
   200  		return out, nil
   201  	}
   202  
   203  	results, err := lu.lkp.Lookup(ctx, vcursor, ids, vcursor.LookupRowLockShardSession())
   204  	if err != nil {
   205  		return nil, err
   206  	}
   207  	return lu.MapResult(ids, results)
   208  }
   209  
   210  // MapResult implements the LookupPlanable interface
   211  func (lu *ConsistentLookupUnique) MapResult(ids []sqltypes.Value, results []*sqltypes.Result) ([]key.Destination, error) {
   212  	out := make([]key.Destination, 0, len(ids))
   213  	if lu.writeOnly {
   214  		for range ids {
   215  			out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}})
   216  		}
   217  		return out, nil
   218  	}
   219  
   220  	for i, result := range results {
   221  		switch len(result.Rows) {
   222  		case 0:
   223  			out = append(out, key.DestinationNone{})
   224  		case 1:
   225  			rowBytes, err := result.Rows[0][0].ToBytes()
   226  			if err != nil {
   227  				return out, err
   228  			}
   229  			out = append(out, key.DestinationKeyspaceID(rowBytes))
   230  		default:
   231  			return nil, fmt.Errorf("Lookup.Map: unexpected multiple results from vindex %s: %v", lu.lkp.Table, ids[i])
   232  		}
   233  	}
   234  	return out, nil
   235  }
   236  
   237  // Query implements the LookupPlanable interface
   238  func (lu *ConsistentLookupUnique) Query() (selQuery string, arguments []string) {
   239  	return lu.lkp.query()
   240  }
   241  
   242  // AllowBatch implements the LookupPlanable interface
   243  func (lu *ConsistentLookupUnique) AllowBatch() bool {
   244  	return lu.lkp.BatchLookup
   245  }
   246  
   247  func (lu *ConsistentLookupUnique) AutoCommitEnabled() bool {
   248  	return lu.lkp.Autocommit
   249  }
   250  
   251  //====================================================================
   252  
   253  // clCommon defines a vindex that uses a lookup table.
   254  // The table is expected to define the id column as unique. It's
   255  // Unique and a Lookup.
   256  type clCommon struct {
   257  	name         string
   258  	writeOnly    bool
   259  	lkp          lookupInternal
   260  	keyspace     string
   261  	ownerTable   string
   262  	ownerColumns []string
   263  
   264  	lockLookupQuery   string
   265  	lockOwnerQuery    string
   266  	insertLookupQuery string
   267  	updateLookupQuery string
   268  }
   269  
   270  // newCLCommon is commone code for the consistent lookup vindexes.
   271  func newCLCommon(name string, m map[string]string) (*clCommon, error) {
   272  	lu := &clCommon{name: name}
   273  	var err error
   274  	lu.writeOnly, err = boolFromMap(m, "write_only")
   275  	if err != nil {
   276  		return nil, err
   277  	}
   278  
   279  	if err := lu.lkp.Init(m, false /* autocommit */, false /* upsert */, false /* multiShardAutocommit */); err != nil {
   280  		return nil, err
   281  	}
   282  	return lu, nil
   283  }
   284  
   285  func (lu *clCommon) SetOwnerInfo(keyspace, table string, cols []sqlparser.IdentifierCI) error {
   286  	lu.keyspace = keyspace
   287  	lu.ownerTable = sqlparser.String(sqlparser.NewIdentifierCS(table))
   288  	if len(cols) != len(lu.lkp.FromColumns) {
   289  		return fmt.Errorf("owner table column count does not match vindex %s", lu.name)
   290  	}
   291  	lu.ownerColumns = make([]string, len(cols))
   292  	for i, col := range cols {
   293  		lu.ownerColumns[i] = col.String()
   294  	}
   295  	lu.lockLookupQuery = lu.generateLockLookup()
   296  	lu.lockOwnerQuery = lu.generateLockOwner()
   297  	lu.insertLookupQuery = lu.generateInsertLookup()
   298  	lu.updateLookupQuery = lu.generateUpdateLookup()
   299  	return nil
   300  }
   301  
   302  // String returns the name of the vindex.
   303  func (lu *clCommon) String() string {
   304  	return lu.name
   305  }
   306  
   307  // Verify returns true if ids maps to ksids.
   308  func (lu *clCommon) Verify(ctx context.Context, vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) {
   309  	if lu.writeOnly {
   310  		out := make([]bool, len(ids))
   311  		for i := range ids {
   312  			out[i] = true
   313  		}
   314  		return out, nil
   315  	}
   316  	return lu.lkp.VerifyCustom(ctx, vcursor, ids, ksidsToValues(ksids), vtgate.CommitOrder_PRE)
   317  }
   318  
   319  // Create reserves the id by inserting it into the vindex table.
   320  func (lu *clCommon) Create(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte, ignoreMode bool) error {
   321  	origErr := lu.lkp.createCustom(ctx, vcursor, rowsColValues, ksidsToValues(ksids), ignoreMode, vtgatepb.CommitOrder_PRE)
   322  	if origErr == nil {
   323  		return nil
   324  	}
   325  	// Try and convert the error to a MySQL error
   326  	sqlErr, isSQLErr := mysql.NewSQLErrorFromError(origErr).(*mysql.SQLError)
   327  	// If it is a MySQL error and its code is of duplicate entry, then we would like to continue
   328  	// Otherwise, we return the error
   329  	if !(isSQLErr && sqlErr != nil && sqlErr.Number() == mysql.ERDupEntry) {
   330  		return origErr
   331  	}
   332  	for i, row := range rowsColValues {
   333  		if err := lu.handleDup(ctx, vcursor, row, ksids[i], origErr); err != nil {
   334  			return err
   335  		}
   336  	}
   337  	return nil
   338  }
   339  
   340  func (lu *clCommon) handleDup(ctx context.Context, vcursor VCursor, values []sqltypes.Value, ksid []byte, dupError error) error {
   341  	bindVars := make(map[string]*querypb.BindVariable, len(values))
   342  	for colnum, val := range values {
   343  		bindVars[lu.lkp.FromColumns[colnum]] = sqltypes.ValueBindVariable(val)
   344  	}
   345  	bindVars[lu.lkp.To] = sqltypes.BytesBindVariable(ksid)
   346  
   347  	// Lock the lookup row using pre priority.
   348  	qr, err := vcursor.Execute(ctx, "VindexCreate", lu.lockLookupQuery, bindVars, false /* rollbackOnError */, vtgatepb.CommitOrder_PRE)
   349  	if err != nil {
   350  		return err
   351  	}
   352  	switch len(qr.Rows) {
   353  	case 0:
   354  		if _, err := vcursor.Execute(ctx, "VindexCreate", lu.insertLookupQuery, bindVars, true /* rollbackOnError */, vtgatepb.CommitOrder_PRE); err != nil {
   355  			return err
   356  		}
   357  	case 1:
   358  		existingksid, err := qr.Rows[0][0].ToBytes()
   359  		if err != nil {
   360  			return err
   361  		}
   362  		// Lock the target row using normal transaction priority.
   363  		// TODO: context needs to be passed on.
   364  		qr, err = vcursor.ExecuteKeyspaceID(context.Background(), lu.keyspace, existingksid, lu.lockOwnerQuery, bindVars, false /* rollbackOnError */, false /* autocommit */)
   365  		if err != nil {
   366  			return err
   367  		}
   368  		if len(qr.Rows) >= 1 {
   369  			return dupError
   370  		}
   371  		if bytes.Equal(existingksid, ksid) {
   372  			return nil
   373  		}
   374  		if _, err := vcursor.Execute(ctx, "VindexCreate", lu.updateLookupQuery, bindVars, true /* rollbackOnError */, vtgatepb.CommitOrder_PRE); err != nil {
   375  			return err
   376  		}
   377  	default:
   378  		return fmt.Errorf("unexpected rows: %v from consistent lookup vindex", qr.Rows)
   379  	}
   380  	return nil
   381  }
   382  
   383  // Delete deletes the entry from the vindex table.
   384  func (lu *clCommon) Delete(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value, ksid []byte) error {
   385  	return lu.lkp.Delete(ctx, vcursor, rowsColValues, sqltypes.MakeTrusted(sqltypes.VarBinary, ksid), vtgatepb.CommitOrder_POST)
   386  }
   387  
   388  // Update updates the entry in the vindex table.
   389  func (lu *clCommon) Update(ctx context.Context, vcursor VCursor, oldValues []sqltypes.Value, ksid []byte, newValues []sqltypes.Value) error {
   390  	equal := true
   391  	for i := range oldValues {
   392  		// TODO(king-11) make collation aware
   393  		result, err := evalengine.NullsafeCompare(oldValues[i], newValues[i], collations.Unknown)
   394  		// errors from NullsafeCompare can be ignored. if they are real problems, we'll see them in the Create/Update
   395  		if err != nil || result != 0 {
   396  			equal = false
   397  			break
   398  		}
   399  	}
   400  	if equal {
   401  		return nil
   402  	}
   403  	if err := lu.Delete(ctx, vcursor, [][]sqltypes.Value{oldValues}, ksid); err != nil {
   404  		return err
   405  	}
   406  	return lu.Create(ctx, vcursor, [][]sqltypes.Value{newValues}, [][]byte{ksid}, false /* ignoreMode */)
   407  }
   408  
   409  // MarshalJSON returns a JSON representation of clCommon.
   410  func (lu *clCommon) MarshalJSON() ([]byte, error) {
   411  	return json.Marshal(lu.lkp)
   412  }
   413  
   414  func (lu *clCommon) generateLockLookup() string {
   415  	var buf bytes.Buffer
   416  	fmt.Fprintf(&buf, "select %s from %s", lu.lkp.To, lu.lkp.Table)
   417  	lu.addWhere(&buf, lu.lkp.FromColumns)
   418  	fmt.Fprintf(&buf, " for update")
   419  	return buf.String()
   420  }
   421  
   422  func (lu *clCommon) generateLockOwner() string {
   423  	var buf bytes.Buffer
   424  	fmt.Fprintf(&buf, "select %s from %s", lu.ownerColumns[0], lu.ownerTable)
   425  	lu.addWhere(&buf, lu.ownerColumns)
   426  	// We can lock in share mode because we only want to check
   427  	// if the row exists. We still need to lock to make us wait
   428  	// in case a previous transaction is creating it.
   429  	fmt.Fprintf(&buf, " lock in share mode")
   430  	return buf.String()
   431  }
   432  
   433  func (lu *clCommon) generateInsertLookup() string {
   434  	var buf bytes.Buffer
   435  	fmt.Fprintf(&buf, "insert into %s(", lu.lkp.Table)
   436  	for _, col := range lu.lkp.FromColumns {
   437  		fmt.Fprintf(&buf, "%s, ", col)
   438  	}
   439  	fmt.Fprintf(&buf, "%s) values(", lu.lkp.To)
   440  	for _, col := range lu.lkp.FromColumns {
   441  		fmt.Fprintf(&buf, ":%s, ", col)
   442  	}
   443  	fmt.Fprintf(&buf, ":%s)", lu.lkp.To)
   444  	return buf.String()
   445  }
   446  
   447  func (lu *clCommon) generateUpdateLookup() string {
   448  	var buf bytes.Buffer
   449  	fmt.Fprintf(&buf, "update %s set %s=:%s", lu.lkp.Table, lu.lkp.To, lu.lkp.To)
   450  	lu.addWhere(&buf, lu.lkp.FromColumns)
   451  	return buf.String()
   452  }
   453  
   454  func (lu *clCommon) addWhere(buf *bytes.Buffer, cols []string) {
   455  	buf.WriteString(" where ")
   456  	for colIdx, column := range cols {
   457  		if colIdx != 0 {
   458  			buf.WriteString(" and ")
   459  		}
   460  		buf.WriteString(column + " = :" + lu.lkp.FromColumns[colIdx])
   461  	}
   462  }
   463  
   464  // GetCommitOrder implements the LookupPlanable interface
   465  func (lu *clCommon) GetCommitOrder() vtgatepb.CommitOrder {
   466  	return vtgatepb.CommitOrder_PRE
   467  }
   468  
   469  // IsBackfilling implements the LookupBackfill interface
   470  func (lu *ConsistentLookupUnique) IsBackfilling() bool {
   471  	return lu.writeOnly
   472  }