vitess.io/vitess@v0.16.2/go/vt/vtgate/vindexes/consistent_lookup.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vindexes 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "fmt" 24 25 "vitess.io/vitess/go/mysql" 26 "vitess.io/vitess/go/mysql/collations" 27 "vitess.io/vitess/go/vt/vtgate/evalengine" 28 29 "vitess.io/vitess/go/sqltypes" 30 "vitess.io/vitess/go/vt/key" 31 querypb "vitess.io/vitess/go/vt/proto/query" 32 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 33 "vitess.io/vitess/go/vt/proto/vtgate" 34 vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" 35 "vitess.io/vitess/go/vt/sqlparser" 36 ) 37 38 var ( 39 _ SingleColumn = (*ConsistentLookupUnique)(nil) 40 _ Lookup = (*ConsistentLookupUnique)(nil) 41 _ WantOwnerInfo = (*ConsistentLookupUnique)(nil) 42 _ LookupPlanable = (*ConsistentLookupUnique)(nil) 43 _ SingleColumn = (*ConsistentLookup)(nil) 44 _ Lookup = (*ConsistentLookup)(nil) 45 _ WantOwnerInfo = (*ConsistentLookup)(nil) 46 _ LookupPlanable = (*ConsistentLookup)(nil) 47 ) 48 49 func init() { 50 Register("consistent_lookup", NewConsistentLookup) 51 Register("consistent_lookup_unique", NewConsistentLookupUnique) 52 } 53 54 // ConsistentLookup is a non-unique lookup vindex that can stay 55 // consistent with respect to its owner table. 56 type ConsistentLookup struct { 57 *clCommon 58 } 59 60 // NewConsistentLookup creates a ConsistentLookup vindex. 61 // The supplied map has the following required fields: 62 // 63 // table: name of the backing table. It can be qualified by the keyspace. 64 // from: list of columns in the table that have the 'from' values of the lookup vindex. 65 // to: The 'to' column name of the table. 66 func NewConsistentLookup(name string, m map[string]string) (Vindex, error) { 67 clc, err := newCLCommon(name, m) 68 if err != nil { 69 return nil, err 70 } 71 return &ConsistentLookup{clCommon: clc}, nil 72 } 73 74 // Cost returns the cost of this vindex as 20. 75 func (lu *ConsistentLookup) Cost() int { 76 return 20 77 } 78 79 // IsUnique returns false since the Vindex is non unique. 80 func (lu *ConsistentLookup) IsUnique() bool { 81 return false 82 } 83 84 // NeedsVCursor satisfies the Vindex interface. 85 func (lu *ConsistentLookup) NeedsVCursor() bool { 86 return true 87 } 88 89 // Map can map ids to key.Destination objects. 90 func (lu *ConsistentLookup) Map(ctx context.Context, vcursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) { 91 out := make([]key.Destination, 0, len(ids)) 92 if lu.writeOnly { 93 for range ids { 94 out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}}) 95 } 96 return out, nil 97 } 98 99 // if ignore_nulls is set and the query is about single null value, then fallback to all shards 100 if len(ids) == 1 && ids[0].IsNull() && lu.lkp.IgnoreNulls { 101 for range ids { 102 out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}}) 103 } 104 return out, nil 105 } 106 107 results, err := lu.lkp.Lookup(ctx, vcursor, ids, vcursor.LookupRowLockShardSession()) 108 if err != nil { 109 return nil, err 110 } 111 return lu.MapResult(ids, results) 112 } 113 114 // MapResult implements the LookupPlanable interface 115 func (lu *ConsistentLookup) MapResult(ids []sqltypes.Value, results []*sqltypes.Result) ([]key.Destination, error) { 116 out := make([]key.Destination, 0, len(ids)) 117 if lu.writeOnly { 118 for range ids { 119 out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}}) 120 } 121 return out, nil 122 } 123 for _, result := range results { 124 if len(result.Rows) == 0 { 125 out = append(out, key.DestinationNone{}) 126 continue 127 } 128 ksids := make([][]byte, 0, len(result.Rows)) 129 for _, row := range result.Rows { 130 rowBytes, err := row[0].ToBytes() 131 if err != nil { 132 return nil, err 133 } 134 ksids = append(ksids, rowBytes) 135 } 136 out = append(out, key.DestinationKeyspaceIDs(ksids)) 137 } 138 return out, nil 139 } 140 141 // Query implements the LookupPlanable interface 142 func (lu *ConsistentLookup) Query() (selQuery string, arguments []string) { 143 return lu.lkp.query() 144 } 145 146 // AllowBatch implements the LookupPlanable interface 147 func (lu *ConsistentLookup) AllowBatch() bool { 148 return lu.lkp.BatchLookup 149 } 150 151 func (lu *ConsistentLookup) AutoCommitEnabled() bool { 152 return lu.lkp.Autocommit 153 } 154 155 //==================================================================== 156 157 // ConsistentLookupUnique defines a vindex that uses a lookup table. 158 // The table is expected to define the id column as unique. It's 159 // Unique and a Lookup. 160 type ConsistentLookupUnique struct { 161 *clCommon 162 } 163 164 // NewConsistentLookupUnique creates a ConsistentLookupUnique vindex. 165 // The supplied map has the following required fields: 166 // 167 // table: name of the backing table. It can be qualified by the keyspace. 168 // from: list of columns in the table that have the 'from' values of the lookup vindex. 169 // to: The 'to' column name of the table. 170 func NewConsistentLookupUnique(name string, m map[string]string) (Vindex, error) { 171 clc, err := newCLCommon(name, m) 172 if err != nil { 173 return nil, err 174 } 175 return &ConsistentLookupUnique{clCommon: clc}, nil 176 } 177 178 // Cost returns the cost of this vindex as 10. 179 func (lu *ConsistentLookupUnique) Cost() int { 180 return 10 181 } 182 183 // IsUnique returns true since the Vindex is unique. 184 func (lu *ConsistentLookupUnique) IsUnique() bool { 185 return true 186 } 187 188 // NeedsVCursor satisfies the Vindex interface. 189 func (lu *ConsistentLookupUnique) NeedsVCursor() bool { 190 return true 191 } 192 193 // Map can map ids to key.Destination objects. 194 func (lu *ConsistentLookupUnique) Map(ctx context.Context, vcursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) { 195 out := make([]key.Destination, 0, len(ids)) 196 if lu.writeOnly { 197 for range ids { 198 out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}}) 199 } 200 return out, nil 201 } 202 203 results, err := lu.lkp.Lookup(ctx, vcursor, ids, vcursor.LookupRowLockShardSession()) 204 if err != nil { 205 return nil, err 206 } 207 return lu.MapResult(ids, results) 208 } 209 210 // MapResult implements the LookupPlanable interface 211 func (lu *ConsistentLookupUnique) MapResult(ids []sqltypes.Value, results []*sqltypes.Result) ([]key.Destination, error) { 212 out := make([]key.Destination, 0, len(ids)) 213 if lu.writeOnly { 214 for range ids { 215 out = append(out, key.DestinationKeyRange{KeyRange: &topodatapb.KeyRange{}}) 216 } 217 return out, nil 218 } 219 220 for i, result := range results { 221 switch len(result.Rows) { 222 case 0: 223 out = append(out, key.DestinationNone{}) 224 case 1: 225 rowBytes, err := result.Rows[0][0].ToBytes() 226 if err != nil { 227 return out, err 228 } 229 out = append(out, key.DestinationKeyspaceID(rowBytes)) 230 default: 231 return nil, fmt.Errorf("Lookup.Map: unexpected multiple results from vindex %s: %v", lu.lkp.Table, ids[i]) 232 } 233 } 234 return out, nil 235 } 236 237 // Query implements the LookupPlanable interface 238 func (lu *ConsistentLookupUnique) Query() (selQuery string, arguments []string) { 239 return lu.lkp.query() 240 } 241 242 // AllowBatch implements the LookupPlanable interface 243 func (lu *ConsistentLookupUnique) AllowBatch() bool { 244 return lu.lkp.BatchLookup 245 } 246 247 func (lu *ConsistentLookupUnique) AutoCommitEnabled() bool { 248 return lu.lkp.Autocommit 249 } 250 251 //==================================================================== 252 253 // clCommon defines a vindex that uses a lookup table. 254 // The table is expected to define the id column as unique. It's 255 // Unique and a Lookup. 256 type clCommon struct { 257 name string 258 writeOnly bool 259 lkp lookupInternal 260 keyspace string 261 ownerTable string 262 ownerColumns []string 263 264 lockLookupQuery string 265 lockOwnerQuery string 266 insertLookupQuery string 267 updateLookupQuery string 268 } 269 270 // newCLCommon is commone code for the consistent lookup vindexes. 271 func newCLCommon(name string, m map[string]string) (*clCommon, error) { 272 lu := &clCommon{name: name} 273 var err error 274 lu.writeOnly, err = boolFromMap(m, "write_only") 275 if err != nil { 276 return nil, err 277 } 278 279 if err := lu.lkp.Init(m, false /* autocommit */, false /* upsert */, false /* multiShardAutocommit */); err != nil { 280 return nil, err 281 } 282 return lu, nil 283 } 284 285 func (lu *clCommon) SetOwnerInfo(keyspace, table string, cols []sqlparser.IdentifierCI) error { 286 lu.keyspace = keyspace 287 lu.ownerTable = sqlparser.String(sqlparser.NewIdentifierCS(table)) 288 if len(cols) != len(lu.lkp.FromColumns) { 289 return fmt.Errorf("owner table column count does not match vindex %s", lu.name) 290 } 291 lu.ownerColumns = make([]string, len(cols)) 292 for i, col := range cols { 293 lu.ownerColumns[i] = col.String() 294 } 295 lu.lockLookupQuery = lu.generateLockLookup() 296 lu.lockOwnerQuery = lu.generateLockOwner() 297 lu.insertLookupQuery = lu.generateInsertLookup() 298 lu.updateLookupQuery = lu.generateUpdateLookup() 299 return nil 300 } 301 302 // String returns the name of the vindex. 303 func (lu *clCommon) String() string { 304 return lu.name 305 } 306 307 // Verify returns true if ids maps to ksids. 308 func (lu *clCommon) Verify(ctx context.Context, vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { 309 if lu.writeOnly { 310 out := make([]bool, len(ids)) 311 for i := range ids { 312 out[i] = true 313 } 314 return out, nil 315 } 316 return lu.lkp.VerifyCustom(ctx, vcursor, ids, ksidsToValues(ksids), vtgate.CommitOrder_PRE) 317 } 318 319 // Create reserves the id by inserting it into the vindex table. 320 func (lu *clCommon) Create(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte, ignoreMode bool) error { 321 origErr := lu.lkp.createCustom(ctx, vcursor, rowsColValues, ksidsToValues(ksids), ignoreMode, vtgatepb.CommitOrder_PRE) 322 if origErr == nil { 323 return nil 324 } 325 // Try and convert the error to a MySQL error 326 sqlErr, isSQLErr := mysql.NewSQLErrorFromError(origErr).(*mysql.SQLError) 327 // If it is a MySQL error and its code is of duplicate entry, then we would like to continue 328 // Otherwise, we return the error 329 if !(isSQLErr && sqlErr != nil && sqlErr.Number() == mysql.ERDupEntry) { 330 return origErr 331 } 332 for i, row := range rowsColValues { 333 if err := lu.handleDup(ctx, vcursor, row, ksids[i], origErr); err != nil { 334 return err 335 } 336 } 337 return nil 338 } 339 340 func (lu *clCommon) handleDup(ctx context.Context, vcursor VCursor, values []sqltypes.Value, ksid []byte, dupError error) error { 341 bindVars := make(map[string]*querypb.BindVariable, len(values)) 342 for colnum, val := range values { 343 bindVars[lu.lkp.FromColumns[colnum]] = sqltypes.ValueBindVariable(val) 344 } 345 bindVars[lu.lkp.To] = sqltypes.BytesBindVariable(ksid) 346 347 // Lock the lookup row using pre priority. 348 qr, err := vcursor.Execute(ctx, "VindexCreate", lu.lockLookupQuery, bindVars, false /* rollbackOnError */, vtgatepb.CommitOrder_PRE) 349 if err != nil { 350 return err 351 } 352 switch len(qr.Rows) { 353 case 0: 354 if _, err := vcursor.Execute(ctx, "VindexCreate", lu.insertLookupQuery, bindVars, true /* rollbackOnError */, vtgatepb.CommitOrder_PRE); err != nil { 355 return err 356 } 357 case 1: 358 existingksid, err := qr.Rows[0][0].ToBytes() 359 if err != nil { 360 return err 361 } 362 // Lock the target row using normal transaction priority. 363 // TODO: context needs to be passed on. 364 qr, err = vcursor.ExecuteKeyspaceID(context.Background(), lu.keyspace, existingksid, lu.lockOwnerQuery, bindVars, false /* rollbackOnError */, false /* autocommit */) 365 if err != nil { 366 return err 367 } 368 if len(qr.Rows) >= 1 { 369 return dupError 370 } 371 if bytes.Equal(existingksid, ksid) { 372 return nil 373 } 374 if _, err := vcursor.Execute(ctx, "VindexCreate", lu.updateLookupQuery, bindVars, true /* rollbackOnError */, vtgatepb.CommitOrder_PRE); err != nil { 375 return err 376 } 377 default: 378 return fmt.Errorf("unexpected rows: %v from consistent lookup vindex", qr.Rows) 379 } 380 return nil 381 } 382 383 // Delete deletes the entry from the vindex table. 384 func (lu *clCommon) Delete(ctx context.Context, vcursor VCursor, rowsColValues [][]sqltypes.Value, ksid []byte) error { 385 return lu.lkp.Delete(ctx, vcursor, rowsColValues, sqltypes.MakeTrusted(sqltypes.VarBinary, ksid), vtgatepb.CommitOrder_POST) 386 } 387 388 // Update updates the entry in the vindex table. 389 func (lu *clCommon) Update(ctx context.Context, vcursor VCursor, oldValues []sqltypes.Value, ksid []byte, newValues []sqltypes.Value) error { 390 equal := true 391 for i := range oldValues { 392 // TODO(king-11) make collation aware 393 result, err := evalengine.NullsafeCompare(oldValues[i], newValues[i], collations.Unknown) 394 // errors from NullsafeCompare can be ignored. if they are real problems, we'll see them in the Create/Update 395 if err != nil || result != 0 { 396 equal = false 397 break 398 } 399 } 400 if equal { 401 return nil 402 } 403 if err := lu.Delete(ctx, vcursor, [][]sqltypes.Value{oldValues}, ksid); err != nil { 404 return err 405 } 406 return lu.Create(ctx, vcursor, [][]sqltypes.Value{newValues}, [][]byte{ksid}, false /* ignoreMode */) 407 } 408 409 // MarshalJSON returns a JSON representation of clCommon. 410 func (lu *clCommon) MarshalJSON() ([]byte, error) { 411 return json.Marshal(lu.lkp) 412 } 413 414 func (lu *clCommon) generateLockLookup() string { 415 var buf bytes.Buffer 416 fmt.Fprintf(&buf, "select %s from %s", lu.lkp.To, lu.lkp.Table) 417 lu.addWhere(&buf, lu.lkp.FromColumns) 418 fmt.Fprintf(&buf, " for update") 419 return buf.String() 420 } 421 422 func (lu *clCommon) generateLockOwner() string { 423 var buf bytes.Buffer 424 fmt.Fprintf(&buf, "select %s from %s", lu.ownerColumns[0], lu.ownerTable) 425 lu.addWhere(&buf, lu.ownerColumns) 426 // We can lock in share mode because we only want to check 427 // if the row exists. We still need to lock to make us wait 428 // in case a previous transaction is creating it. 429 fmt.Fprintf(&buf, " lock in share mode") 430 return buf.String() 431 } 432 433 func (lu *clCommon) generateInsertLookup() string { 434 var buf bytes.Buffer 435 fmt.Fprintf(&buf, "insert into %s(", lu.lkp.Table) 436 for _, col := range lu.lkp.FromColumns { 437 fmt.Fprintf(&buf, "%s, ", col) 438 } 439 fmt.Fprintf(&buf, "%s) values(", lu.lkp.To) 440 for _, col := range lu.lkp.FromColumns { 441 fmt.Fprintf(&buf, ":%s, ", col) 442 } 443 fmt.Fprintf(&buf, ":%s)", lu.lkp.To) 444 return buf.String() 445 } 446 447 func (lu *clCommon) generateUpdateLookup() string { 448 var buf bytes.Buffer 449 fmt.Fprintf(&buf, "update %s set %s=:%s", lu.lkp.Table, lu.lkp.To, lu.lkp.To) 450 lu.addWhere(&buf, lu.lkp.FromColumns) 451 return buf.String() 452 } 453 454 func (lu *clCommon) addWhere(buf *bytes.Buffer, cols []string) { 455 buf.WriteString(" where ") 456 for colIdx, column := range cols { 457 if colIdx != 0 { 458 buf.WriteString(" and ") 459 } 460 buf.WriteString(column + " = :" + lu.lkp.FromColumns[colIdx]) 461 } 462 } 463 464 // GetCommitOrder implements the LookupPlanable interface 465 func (lu *clCommon) GetCommitOrder() vtgatepb.CommitOrder { 466 return vtgatepb.CommitOrder_PRE 467 } 468 469 // IsBackfilling implements the LookupBackfill interface 470 func (lu *ConsistentLookupUnique) IsBackfilling() bool { 471 return lu.writeOnly 472 }