vitess.io/vitess@v0.16.2/go/vt/vtgate/vindexes/vschema.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vindexes 18 19 import ( 20 "encoding/hex" 21 "encoding/json" 22 "fmt" 23 "os" 24 "sort" 25 "strings" 26 27 "vitess.io/vitess/go/sqlescape" 28 vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" 29 "vitess.io/vitess/go/vt/vterrors" 30 31 "vitess.io/vitess/go/json2" 32 "vitess.io/vitess/go/sqltypes" 33 "vitess.io/vitess/go/vt/sqlparser" 34 35 querypb "vitess.io/vitess/go/vt/proto/query" 36 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 37 vschemapb "vitess.io/vitess/go/vt/proto/vschema" 38 ) 39 40 // TabletTypeSuffix maps the tablet type to its suffix string. 41 var TabletTypeSuffix = map[topodatapb.TabletType]string{ 42 0: "@unknown", 43 1: "@primary", 44 2: "@replica", 45 3: "@rdonly", 46 4: "@spare", 47 5: "@experimental", 48 6: "@backup", 49 7: "@restore", 50 8: "@drained", 51 } 52 53 // The following constants represent table types. 54 const ( 55 TypeSequence = "sequence" 56 TypeReference = "reference" 57 ) 58 59 // VSchema represents the denormalized version of SrvVSchema, 60 // used for building routing plans. 61 type VSchema struct { 62 RoutingRules map[string]*RoutingRule `json:"routing_rules"` 63 64 // uniqueTables contains the name of all tables in all keyspaces. if the table is uniquely named, the value will 65 // be the name of the keyspace where this table exists. if multiple keyspaces have a table with the same name, the 66 // value will be a `nil` value 67 globalTables map[string]*Table 68 uniqueVindexes map[string]Vindex 69 Keyspaces map[string]*KeyspaceSchema `json:"keyspaces"` 70 ShardRoutingRules map[string]string `json:"shard_routing_rules"` 71 } 72 73 // RoutingRule represents one routing rule. 74 type RoutingRule struct { 75 Tables []*Table 76 Error error 77 } 78 79 // MarshalJSON returns a JSON representation of Column. 80 func (rr *RoutingRule) MarshalJSON() ([]byte, error) { 81 if rr.Error != nil { 82 return json.Marshal(rr.Error.Error()) 83 } 84 tables := make([]string, 0, len(rr.Tables)) 85 for _, t := range rr.Tables { 86 tables = append(tables, t.String()) 87 } 88 89 return json.Marshal(tables) 90 } 91 92 // Table represents a table in VSchema. 93 type Table struct { 94 Type string `json:"type,omitempty"` 95 Name sqlparser.IdentifierCS `json:"name"` 96 Keyspace *Keyspace `json:"-"` 97 ColumnVindexes []*ColumnVindex `json:"column_vindexes,omitempty"` 98 Ordered []*ColumnVindex `json:"ordered,omitempty"` 99 Owned []*ColumnVindex `json:"owned,omitempty"` 100 AutoIncrement *AutoIncrement `json:"auto_increment,omitempty"` 101 Columns []Column `json:"columns,omitempty"` 102 Pinned []byte `json:"pinned,omitempty"` 103 ColumnListAuthoritative bool `json:"column_list_authoritative,omitempty"` 104 // ReferencedBy is an inverse mapping of tables in other keyspaces that 105 // reference this table via Source. 106 // 107 // This is useful in route-planning for quickly selecting the optimal route 108 // when JOIN-ing a reference table to a sharded table. 109 ReferencedBy map[string]*Table `json:"-"` 110 // Source is a keyspace-qualified table name that points to the source of a 111 // reference table. Only applicable for tables with Type set to "reference". 112 Source *Source `json:"source,omitempty"` 113 } 114 115 // Keyspace contains the keyspcae info for each Table. 116 type Keyspace struct { 117 Name string 118 Sharded bool 119 } 120 121 // ColumnVindex contains the index info for each index of a table. 122 type ColumnVindex struct { 123 Columns []sqlparser.IdentifierCI `json:"columns"` 124 Type string `json:"type"` 125 Name string `json:"name"` 126 Owned bool `json:"owned,omitempty"` 127 Vindex Vindex `json:"vindex"` 128 isUnique bool 129 cost int 130 partial bool 131 } 132 133 // IsUnique is used to tell whether the ColumnVindex 134 // will return a unique shard value or not when queried with 135 // the given column list 136 func (c *ColumnVindex) IsUnique() bool { 137 return c.isUnique 138 } 139 140 // Cost represents the cost associated with using the 141 // ColumnVindex 142 func (c *ColumnVindex) Cost() int { 143 return c.cost 144 } 145 146 // IsPartialVindex is used to let planner and engine know that this is a composite vindex missing one or more columns 147 func (c *ColumnVindex) IsPartialVindex() bool { 148 return c.partial 149 } 150 151 // Column describes a column. 152 type Column struct { 153 Name sqlparser.IdentifierCI `json:"name"` 154 Type querypb.Type `json:"type"` 155 CollationName string `json:"collation_name"` 156 } 157 158 // MarshalJSON returns a JSON representation of Column. 159 func (col *Column) MarshalJSON() ([]byte, error) { 160 return json.Marshal(struct { 161 Name string `json:"name"` 162 Type string `json:"type,omitempty"` 163 }{ 164 Name: col.Name.String(), 165 Type: querypb.Type_name[int32(col.Type)], 166 }) 167 } 168 169 // KeyspaceSchema contains the schema(table) for a keyspace. 170 type KeyspaceSchema struct { 171 Keyspace *Keyspace 172 Tables map[string]*Table 173 Vindexes map[string]Vindex 174 Views map[string]sqlparser.SelectStatement 175 Error error 176 } 177 178 type ksJSON struct { 179 Sharded bool `json:"sharded,omitempty"` 180 Tables map[string]*Table `json:"tables,omitempty"` 181 Vindexes map[string]Vindex `json:"vindexes,omitempty"` 182 Views map[string]string `json:"views,omitempty"` 183 Error string `json:"error,omitempty"` 184 } 185 186 // MarshalJSON returns a JSON representation of KeyspaceSchema. 187 func (ks *KeyspaceSchema) MarshalJSON() ([]byte, error) { 188 ksJ := ksJSON{ 189 Sharded: ks.Keyspace.Sharded, 190 Tables: ks.Tables, 191 Vindexes: ks.Vindexes, 192 } 193 if ks.Error != nil { 194 ksJ.Error = ks.Error.Error() 195 } 196 if len(ks.Views) > 0 { 197 ksJ.Views = make(map[string]string, len(ks.Views)) 198 } 199 for view, def := range ks.Views { 200 ksJ.Views[view] = sqlparser.String(def) 201 } 202 203 return json.Marshal(ksJ) 204 } 205 206 // AutoIncrement contains the auto-inc information for a table. 207 type AutoIncrement struct { 208 Column sqlparser.IdentifierCI `json:"column"` 209 Sequence *Table `json:"sequence"` 210 } 211 212 type Source struct { 213 sqlparser.TableName 214 } 215 216 func (source *Source) String() string { 217 buf := sqlparser.NewTrackedBuffer(nil) 218 source.Format(buf) 219 return buf.String() 220 } 221 222 // BuildVSchema builds a VSchema from a SrvVSchema. 223 func BuildVSchema(source *vschemapb.SrvVSchema) (vschema *VSchema) { 224 vschema = &VSchema{ 225 RoutingRules: make(map[string]*RoutingRule), 226 globalTables: make(map[string]*Table), 227 uniqueVindexes: make(map[string]Vindex), 228 Keyspaces: make(map[string]*KeyspaceSchema), 229 } 230 buildKeyspaces(source, vschema) 231 buildReferences(source, vschema) 232 buildGlobalTables(source, vschema) 233 resolveAutoIncrement(source, vschema) 234 addDual(vschema) 235 buildRoutingRule(source, vschema) 236 buildShardRoutingRule(source, vschema) 237 return vschema 238 } 239 240 // BuildKeyspaceSchema builds the vschema portion for one keyspace. 241 // The build ignores sequence references because those dependencies can 242 // go cross-keyspace. 243 func BuildKeyspaceSchema(input *vschemapb.Keyspace, keyspace string) (*KeyspaceSchema, error) { 244 if input == nil { 245 input = &vschemapb.Keyspace{} 246 } 247 formal := &vschemapb.SrvVSchema{ 248 Keyspaces: map[string]*vschemapb.Keyspace{ 249 keyspace: input, 250 }, 251 } 252 vschema := &VSchema{ 253 globalTables: make(map[string]*Table), 254 uniqueVindexes: make(map[string]Vindex), 255 Keyspaces: make(map[string]*KeyspaceSchema), 256 } 257 buildKeyspaces(formal, vschema) 258 err := vschema.Keyspaces[keyspace].Error 259 return vschema.Keyspaces[keyspace], err 260 } 261 262 // ValidateKeyspace ensures that the keyspace vschema is valid. 263 // External references (like sequence) are not validated. 264 func ValidateKeyspace(input *vschemapb.Keyspace) error { 265 _, err := BuildKeyspaceSchema(input, "") 266 return err 267 } 268 269 func buildKeyspaces(source *vschemapb.SrvVSchema, vschema *VSchema) { 270 for ksname, ks := range source.Keyspaces { 271 ksvschema := &KeyspaceSchema{ 272 Keyspace: &Keyspace{ 273 Name: ksname, 274 Sharded: ks.Sharded, 275 }, 276 Tables: make(map[string]*Table), 277 Vindexes: make(map[string]Vindex), 278 } 279 vschema.Keyspaces[ksname] = ksvschema 280 ksvschema.Error = buildTables(ks, vschema, ksvschema) 281 } 282 } 283 284 func (vschema *VSchema) AddView(ksname string, viewName, query string) error { 285 ks, ok := vschema.Keyspaces[ksname] 286 if !ok { 287 return fmt.Errorf("keyspace %s not found in vschema", ksname) 288 } 289 ast, err := sqlparser.Parse(query) 290 if err != nil { 291 return err 292 } 293 selectStmt, ok := ast.(sqlparser.SelectStatement) 294 if !ok { 295 return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "expected SELECT or UNION query, got %T", ast) 296 } 297 if ks.Views == nil { 298 ks.Views = make(map[string]sqlparser.SelectStatement) 299 } 300 ks.Views[viewName] = selectStmt 301 t := &Table{ 302 Type: "View", 303 Name: sqlparser.NewIdentifierCS(viewName), 304 Keyspace: ks.Keyspace, 305 ColumnListAuthoritative: true, 306 } 307 vschema.addTableName(t) 308 return nil 309 } 310 311 func buildGlobalTables(source *vschemapb.SrvVSchema, vschema *VSchema) { 312 for ksname, ks := range source.Keyspaces { 313 ksvschema := vschema.Keyspaces[ksname] 314 // If the keyspace requires explicit routing, don't include any of 315 // its tables in global tables. 316 if ks.RequireExplicitRouting { 317 continue 318 } 319 buildKeyspaceGlobalTables(vschema, ksvschema) 320 } 321 } 322 323 func buildKeyspaceGlobalTables(vschema *VSchema, ksvschema *KeyspaceSchema) { 324 for tname, t := range ksvschema.Tables { 325 if gt, ok := vschema.globalTables[tname]; ok { 326 // There is already an entry table stored in global tables 327 // with this name. 328 if gt == nil { 329 // Table name is already marked ambiguous, nothing to do. 330 continue 331 } else if t.isReferencedInKeyspace(gt.Keyspace.Name) { 332 // If the stored table refers to this table, store this 333 // table instead. 334 vschema.globalTables[tname] = t 335 } else if gt.isReferencedInKeyspace(t.Keyspace.Name) { 336 // The source of this table is already stored. Do nothing. 337 continue 338 } else { 339 // Otherwise, mark this table name ambiguous. 340 vschema.globalTables[tname] = nil 341 } 342 } else { 343 vschema.globalTables[tname] = t 344 } 345 } 346 } 347 348 func buildReferences(source *vschemapb.SrvVSchema, vschema *VSchema) { 349 for ksname := range source.Keyspaces { 350 ksvschema := vschema.Keyspaces[ksname] 351 if err := buildKeyspaceReferences(vschema, ksvschema); err != nil && ksvschema.Error == nil { 352 ksvschema.Error = err 353 } 354 } 355 } 356 357 func buildKeyspaceReferences(vschema *VSchema, ksvschema *KeyspaceSchema) error { 358 keyspace := ksvschema.Keyspace 359 for tname, t := range ksvschema.Tables { 360 source := t.Source 361 362 if t.Type != TypeReference || source == nil { 363 continue 364 } 365 366 sourceKsname := source.Qualifier.String() 367 368 // Prohibit self-references. 369 if sourceKsname == keyspace.Name { 370 return vterrors.Errorf( 371 vtrpcpb.Code_UNIMPLEMENTED, 372 "source %q may not reference a table in the same keyspace as table: %s", 373 source, 374 tname, 375 ) 376 } 377 378 // Validate that reference can be resolved. 379 _, sourceT, err := vschema.findKeyspaceAndTableBySource(source) 380 if sourceT == nil { 381 return err 382 } 383 384 // Validate source table types. 385 if !(sourceT.Type == "" || sourceT.Type == TypeReference) { 386 return vterrors.Errorf( 387 vtrpcpb.Code_UNIMPLEMENTED, 388 "source %q may not reference a table of type %q: %s", 389 source, 390 sourceT.Type, 391 tname, 392 ) 393 } 394 395 // Update inverse reference table map. 396 if ot := sourceT.getReferenceInKeyspace(keyspace.Name); ot != nil { 397 names := []string{ot.Name.String(), tname} 398 sort.Strings(names) 399 return vterrors.Errorf( 400 vtrpcpb.Code_UNIMPLEMENTED, 401 "source %q may not be referenced more than once per keyspace: %s, %s", 402 source, 403 names[0], 404 names[1], 405 ) 406 } 407 sourceT.addReferenceInKeyspace(keyspace.Name, t) 408 409 // Forbid reference chains. 410 for sourceT.Source != nil { 411 chain := fmt.Sprintf("%s => %s => %s", tname, sourceT, sourceT.Source) 412 413 return vterrors.Errorf( 414 vtrpcpb.Code_UNIMPLEMENTED, 415 "reference chaining is not allowed %s: %s", 416 chain, 417 tname, 418 ) 419 } 420 } 421 422 return nil 423 } 424 425 func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSchema) error { 426 keyspace := ksvschema.Keyspace 427 for vname, vindexInfo := range ks.Vindexes { 428 vindex, err := CreateVindex(vindexInfo.Type, vname, vindexInfo.Params) 429 if err != nil { 430 return err 431 } 432 433 // If the keyspace requires explicit routing, don't include its indexes 434 // in global routing. 435 if !ks.RequireExplicitRouting { 436 if _, ok := vschema.uniqueVindexes[vname]; ok { 437 vschema.uniqueVindexes[vname] = nil 438 } else { 439 vschema.uniqueVindexes[vname] = vindex 440 } 441 } 442 ksvschema.Vindexes[vname] = vindex 443 } 444 for tname, table := range ks.Tables { 445 t := &Table{ 446 Name: sqlparser.NewIdentifierCS(tname), 447 Keyspace: keyspace, 448 ColumnListAuthoritative: table.ColumnListAuthoritative, 449 } 450 switch table.Type { 451 case "": 452 t.Type = table.Type 453 case TypeReference: 454 if table.Source != "" { 455 tableName, err := parseQualifiedTable(table.Source) 456 if err != nil { 457 return vterrors.Errorf( 458 vtrpcpb.Code_INVALID_ARGUMENT, 459 "invalid source %q for reference table: %s; %v", 460 table.Source, 461 tname, 462 err, 463 ) 464 } 465 t.Source = &Source{TableName: tableName} 466 } 467 t.Type = table.Type 468 case TypeSequence: 469 if keyspace.Sharded && table.Pinned == "" { 470 return vterrors.Errorf( 471 vtrpcpb.Code_FAILED_PRECONDITION, 472 "sequence table has to be in an unsharded keyspace or must be pinned: %s", 473 tname, 474 ) 475 } 476 t.Type = table.Type 477 default: 478 return vterrors.Errorf( 479 vtrpcpb.Code_NOT_FOUND, 480 "unidentified table type %s", 481 table.Type, 482 ) 483 } 484 if table.Pinned != "" { 485 decoded, err := hex.DecodeString(table.Pinned) 486 if err != nil { 487 return vterrors.Errorf( 488 vtrpcpb.Code_INVALID_ARGUMENT, 489 "could not decode the keyspace id for pin: %v", 490 err, 491 ) 492 } 493 t.Pinned = decoded 494 } 495 496 // If keyspace is sharded, then any table that's not a reference or pinned must have vindexes. 497 if keyspace.Sharded && t.Type != TypeReference && table.Pinned == "" && len(table.ColumnVindexes) == 0 { 498 return vterrors.Errorf( 499 vtrpcpb.Code_NOT_FOUND, 500 "missing primary col vindex for table: %s", 501 tname, 502 ) 503 } 504 505 // Initialize Columns. 506 colNames := make(map[string]bool) 507 for _, col := range table.Columns { 508 name := sqlparser.NewIdentifierCI(col.Name) 509 if colNames[name.Lowered()] { 510 return vterrors.Errorf( 511 vtrpcpb.Code_INVALID_ARGUMENT, 512 "duplicate column name '%v' for table: %s", 513 name, 514 tname, 515 ) 516 } 517 colNames[name.Lowered()] = true 518 t.Columns = append(t.Columns, Column{Name: name, Type: col.Type}) 519 } 520 521 // Initialize ColumnVindexes. 522 for i, ind := range table.ColumnVindexes { 523 vindexInfo, ok := ks.Vindexes[ind.Name] 524 if !ok { 525 return vterrors.Errorf( 526 vtrpcpb.Code_NOT_FOUND, 527 "vindex %s not found for table %s", 528 ind.Name, 529 tname, 530 ) 531 } 532 vindex := ksvschema.Vindexes[ind.Name] 533 owned := false 534 if _, ok := vindex.(Lookup); ok && vindexInfo.Owner == tname { 535 owned = true 536 } 537 var columns []sqlparser.IdentifierCI 538 if ind.Column != "" { 539 if len(ind.Columns) > 0 { 540 return vterrors.Errorf( 541 vtrpcpb.Code_INVALID_ARGUMENT, 542 "can't use column and columns at the same time in vindex (%s) and table (%s)", 543 ind.Name, 544 tname, 545 ) 546 } 547 columns = []sqlparser.IdentifierCI{sqlparser.NewIdentifierCI(ind.Column)} 548 } else { 549 if len(ind.Columns) == 0 { 550 return vterrors.Errorf( 551 vtrpcpb.Code_INVALID_ARGUMENT, 552 "must specify at least one column for vindex (%s) and table (%s)", 553 ind.Name, 554 tname, 555 ) 556 } 557 for _, indCol := range ind.Columns { 558 columns = append(columns, sqlparser.NewIdentifierCI(indCol)) 559 } 560 } 561 columnVindex := &ColumnVindex{ 562 Columns: columns, 563 Type: vindexInfo.Type, 564 Name: ind.Name, 565 Owned: owned, 566 Vindex: vindex, 567 isUnique: vindex.IsUnique(), 568 cost: vindex.Cost(), 569 } 570 if i == 0 { 571 // Perform Primary vindex check. 572 if !columnVindex.Vindex.IsUnique() { 573 return vterrors.Errorf( 574 vtrpcpb.Code_INVALID_ARGUMENT, 575 "primary vindex %s is not Unique for table %s", 576 ind.Name, 577 tname, 578 ) 579 } 580 if owned { 581 return vterrors.Errorf( 582 vtrpcpb.Code_INVALID_ARGUMENT, 583 "primary vindex %s cannot be owned for table %s", 584 ind.Name, 585 tname, 586 ) 587 } 588 } 589 t.ColumnVindexes = append(t.ColumnVindexes, columnVindex) 590 if owned { 591 if setter, ok := vindex.(WantOwnerInfo); ok { 592 if err := setter.SetOwnerInfo(keyspace.Name, tname, columns); err != nil { 593 return err 594 } 595 } 596 t.Owned = append(t.Owned, columnVindex) 597 } 598 599 mcv, isMultiColumn := vindex.(MultiColumn) 600 if !isMultiColumn { 601 continue 602 } 603 if i != 0 { 604 return vterrors.Errorf( 605 vtrpcpb.Code_UNIMPLEMENTED, 606 "multi-column vindex %s should be a primary vindex for table %s", 607 ind.Name, 608 tname, 609 ) 610 } 611 if !mcv.PartialVindex() { 612 // Partial column selection not allowed. 613 // Do not create subset column vindex. 614 continue 615 } 616 cost := vindex.Cost() 617 for i := len(columns) - 1; i > 0; i-- { 618 columnSubset := columns[:i] 619 cost++ 620 columnVindex = &ColumnVindex{ 621 Columns: columnSubset, 622 Type: vindexInfo.Type, 623 Name: ind.Name, 624 Owned: owned, 625 Vindex: vindex, 626 cost: cost, 627 partial: true, 628 } 629 t.ColumnVindexes = append(t.ColumnVindexes, columnVindex) 630 } 631 } 632 t.Ordered = colVindexSorted(t.ColumnVindexes) 633 634 // Add the table to the map entries. 635 ksvschema.Tables[tname] = t 636 } 637 638 return nil 639 } 640 641 func (vschema *VSchema) addTableName(t *Table) { 642 tname := t.Name.String() 643 if _, ok := vschema.globalTables[tname]; ok { 644 vschema.globalTables[tname] = nil 645 } else { 646 vschema.globalTables[tname] = t 647 } 648 } 649 650 func resolveAutoIncrement(source *vschemapb.SrvVSchema, vschema *VSchema) { 651 for ksname, ks := range source.Keyspaces { 652 ksvschema := vschema.Keyspaces[ksname] 653 for tname, table := range ks.Tables { 654 t := ksvschema.Tables[tname] 655 if t == nil || table.AutoIncrement == nil { 656 continue 657 } 658 seqks, seqtab, err := sqlparser.ParseTable(table.AutoIncrement.Sequence) 659 var seq *Table 660 if err == nil { 661 seq, err = vschema.FindTable(seqks, seqtab) 662 } 663 if err != nil { 664 // Better to remove the table than to leave it partially initialized. 665 delete(ksvschema.Tables, tname) 666 delete(vschema.globalTables, tname) 667 ksvschema.Error = vterrors.Errorf( 668 vtrpcpb.Code_NOT_FOUND, 669 "cannot resolve sequence %s: %s", 670 table.AutoIncrement.Sequence, 671 err.Error(), 672 ) 673 674 continue 675 } 676 t.AutoIncrement = &AutoIncrement{ 677 Column: sqlparser.NewIdentifierCI(table.AutoIncrement.Column), 678 Sequence: seq, 679 } 680 } 681 } 682 } 683 684 // addDual adds dual as a valid table to all keyspaces. 685 // For sharded keyspaces, it gets pinned against keyspace id '0x00'. 686 func addDual(vschema *VSchema) { 687 first := "" 688 for ksname, ks := range vschema.Keyspaces { 689 t := &Table{ 690 Name: sqlparser.NewIdentifierCS("dual"), 691 Keyspace: ks.Keyspace, 692 Type: TypeReference, 693 } 694 ks.Tables["dual"] = t 695 if first == "" || first > ksname { 696 // In case of a reference to dual that's not qualified 697 // by keyspace, we still want to resolve it to one of 698 // the keyspaces. For consistency, we'll always use the 699 // first keyspace by lexical ordering. 700 first = ksname 701 vschema.globalTables["dual"] = t 702 } 703 } 704 } 705 706 // expects table name of the form <keyspace>.<tablename> 707 func escapeQualifiedTable(qualifiedTableName string) (string, error) { 708 keyspace, tableName, err := extractQualifiedTableParts(qualifiedTableName) 709 if err != nil { 710 return "", err 711 } 712 return fmt.Sprintf("%s.%s", 713 // unescape() first in case an already escaped string was passed 714 sqlescape.EscapeID(sqlescape.UnescapeID(keyspace)), 715 sqlescape.EscapeID(sqlescape.UnescapeID(tableName))), nil 716 } 717 718 func extractQualifiedTableParts(qualifiedTableName string) (string, string, error) { 719 // It's possible to have a database or table name with a dot in it, but that's not otherwise supported within vitess today 720 arr := strings.Split(qualifiedTableName, ".") 721 switch len(arr) { 722 case 2: 723 return arr[0], arr[1], nil 724 } 725 // Using fmt.Errorf instead of vterrors here because this error is always wrapped in vterrors. 726 return "", "", fmt.Errorf( 727 "invalid table name: %s, it must be of the qualified form <keyspace_name>.<table_name> (dots are not allowed in either name)", 728 qualifiedTableName, 729 ) 730 } 731 732 func parseQualifiedTable(qualifiedTableName string) (sqlparser.TableName, error) { 733 keyspace, tableName, err := extractQualifiedTableParts(qualifiedTableName) 734 if err != nil { 735 return sqlparser.TableName{}, err 736 } 737 return sqlparser.TableName{ 738 Qualifier: sqlparser.NewIdentifierCS(keyspace), 739 Name: sqlparser.NewIdentifierCS(tableName), 740 }, nil 741 } 742 743 func buildRoutingRule(source *vschemapb.SrvVSchema, vschema *VSchema) { 744 var err error 745 if source.RoutingRules == nil { 746 return 747 } 748 outer: 749 for _, rule := range source.RoutingRules.Rules { 750 rr := &RoutingRule{} 751 if len(rule.ToTables) > 1 { 752 vschema.RoutingRules[rule.FromTable] = &RoutingRule{ 753 Error: vterrors.Errorf( 754 vtrpcpb.Code_INVALID_ARGUMENT, 755 "table %v has more than one target: %v", 756 rule.FromTable, 757 rule.ToTables, 758 ), 759 } 760 continue 761 } 762 for _, toTable := range rule.ToTables { 763 if _, ok := vschema.RoutingRules[rule.FromTable]; ok { 764 vschema.RoutingRules[rule.FromTable] = &RoutingRule{ 765 Error: vterrors.Errorf( 766 vtrpcpb.Code_ALREADY_EXISTS, 767 "duplicate rule for entry %s", 768 rule.FromTable, 769 ), 770 } 771 continue outer 772 } 773 774 // we need to backtick the keyspace and table name before calling ParseTable 775 toTable, err = escapeQualifiedTable(toTable) 776 if err != nil { 777 vschema.RoutingRules[rule.FromTable] = &RoutingRule{ 778 Error: vterrors.Errorf( 779 vtrpcpb.Code_INVALID_ARGUMENT, 780 err.Error(), 781 ), 782 } 783 continue outer 784 } 785 786 toKeyspace, toTableName, err := sqlparser.ParseTable(toTable) 787 788 if err != nil { 789 vschema.RoutingRules[rule.FromTable] = &RoutingRule{ 790 Error: err, 791 } 792 continue outer 793 } 794 if toKeyspace == "" { 795 vschema.RoutingRules[rule.FromTable] = &RoutingRule{ 796 Error: vterrors.Errorf( 797 vtrpcpb.Code_INVALID_ARGUMENT, 798 "table %s must be qualified", 799 toTable, 800 ), 801 } 802 continue outer 803 } 804 t, err := vschema.FindTable(toKeyspace, toTableName) 805 if err != nil { 806 vschema.RoutingRules[rule.FromTable] = &RoutingRule{ 807 Error: err, 808 } 809 continue outer 810 } 811 rr.Tables = append(rr.Tables, t) 812 } 813 vschema.RoutingRules[rule.FromTable] = rr 814 } 815 } 816 817 func buildShardRoutingRule(source *vschemapb.SrvVSchema, vschema *VSchema) { 818 if source.ShardRoutingRules == nil || len(source.ShardRoutingRules.Rules) == 0 { 819 return 820 } 821 vschema.ShardRoutingRules = make(map[string]string) 822 for _, rule := range source.ShardRoutingRules.Rules { 823 vschema.ShardRoutingRules[getShardRoutingRulesKey(rule.FromKeyspace, rule.Shard)] = rule.ToKeyspace 824 } 825 } 826 827 // FindTable returns a pointer to the Table. If a keyspace is specified, only tables 828 // from that keyspace are searched. If the specified keyspace is unsharded 829 // and no tables matched, it's considered valid: FindTable will construct a table 830 // of that name and return it. If no keyspace is specified, then a table is returned 831 // only if its name is unique across all keyspaces. If there is only one 832 // keyspace in the vschema, and it's unsharded, then all table requests are considered 833 // valid and belonging to that keyspace. 834 // FindTable bypasses routing rules and returns at most one table. 835 func (vschema *VSchema) FindTable(keyspace, tablename string) (*Table, error) { 836 t, err := vschema.findTable(keyspace, tablename) 837 if err != nil { 838 return nil, err 839 } 840 if t == nil { 841 return nil, vterrors.NewErrorf( 842 vtrpcpb.Code_NOT_FOUND, 843 vterrors.UnknownTable, 844 "table %s not found", 845 tablename, 846 ) 847 } 848 return t, nil 849 } 850 851 // findTable is like FindTable, but does not return an error if a table is not found. 852 func (vschema *VSchema) findTable(keyspace, tablename string) (*Table, error) { 853 if keyspace == "" { 854 t, ok := vschema.globalTables[tablename] 855 if t == nil { 856 if ok { 857 return nil, vterrors.Errorf( 858 vtrpcpb.Code_FAILED_PRECONDITION, 859 "ambiguous table reference: %s", 860 tablename, 861 ) 862 } 863 if len(vschema.Keyspaces) != 1 { 864 return nil, nil 865 } 866 // Loop happens only once. 867 for _, ks := range vschema.Keyspaces { 868 if ks.Keyspace.Sharded { 869 return nil, nil 870 } 871 return &Table{Name: sqlparser.NewIdentifierCS(tablename), Keyspace: ks.Keyspace}, nil 872 } 873 } 874 keyspace = t.Keyspace.Name 875 } 876 ks, ok := vschema.Keyspaces[keyspace] 877 if !ok { 878 return nil, vterrors.VT05003(keyspace) 879 } 880 table := ks.Tables[tablename] 881 if table == nil { 882 if ks.Keyspace.Sharded { 883 return nil, nil 884 } 885 return &Table{Name: sqlparser.NewIdentifierCS(tablename), Keyspace: ks.Keyspace}, nil 886 } 887 return table, nil 888 } 889 890 // FindRoutedTable finds a table checking the routing rules. 891 func (vschema *VSchema) FindRoutedTable(keyspace, tablename string, tabletType topodatapb.TabletType) (*Table, error) { 892 qualified := tablename 893 if keyspace != "" { 894 qualified = keyspace + "." + tablename 895 } 896 fqtn := qualified + TabletTypeSuffix[tabletType] 897 // First look for a fully qualified table name: keyspace.table@tablet_type. 898 // Then look for one without tablet type: keyspace.table. 899 for _, name := range []string{fqtn, qualified} { 900 rr, ok := vschema.RoutingRules[name] 901 if ok { 902 if rr.Error != nil { 903 return nil, rr.Error 904 } 905 if len(rr.Tables) == 0 { 906 return nil, vterrors.Errorf( 907 vtrpcpb.Code_FAILED_PRECONDITION, 908 "table %s has been disabled", 909 tablename, 910 ) 911 } 912 return rr.Tables[0], nil 913 } 914 } 915 return vschema.findTable(keyspace, tablename) 916 } 917 918 // FindTableOrVindex finds a table or a Vindex by name using Find and FindVindex. 919 func (vschema *VSchema) FindTableOrVindex(keyspace, name string, tabletType topodatapb.TabletType) (*Table, Vindex, error) { 920 tables, err := vschema.FindRoutedTable(keyspace, name, tabletType) 921 if err != nil { 922 return nil, nil, err 923 } 924 if tables != nil { 925 return tables, nil, nil 926 } 927 v, err := vschema.FindVindex(keyspace, name) 928 if err != nil { 929 return nil, nil, err 930 } 931 if v != nil { 932 return nil, v, nil 933 } 934 return nil, nil, NotFoundError{TableName: name} 935 } 936 937 func (vschema *VSchema) FindView(keyspace, name string) sqlparser.SelectStatement { 938 if keyspace == "" { 939 switch { 940 case len(vschema.Keyspaces) == 1: 941 for _, schema := range vschema.Keyspaces { 942 keyspace = schema.Keyspace.Name 943 break 944 } 945 default: 946 t, ok := vschema.globalTables[name] 947 if !ok { 948 return nil 949 } 950 if ok && t == nil { 951 return nil 952 } 953 keyspace = t.Keyspace.Name 954 } 955 } 956 957 ks := vschema.Keyspaces[keyspace] 958 if ks == nil { 959 return nil 960 } 961 962 statement, ok := ks.Views[name] 963 if !ok { 964 return nil 965 } 966 967 // We do this to make sure there is no shared state between uses of this AST 968 statement = sqlparser.CloneSelectStatement(statement) 969 sqlparser.SafeRewrite(statement, nil, func(cursor *sqlparser.Cursor) bool { 970 col, ok := cursor.Node().(*sqlparser.ColName) 971 if ok { 972 cursor.Replace(sqlparser.NewColNameWithQualifier(col.Name.String(), col.Qualifier)) 973 } 974 return true 975 }) 976 return statement 977 } 978 979 func (vschema *VSchema) findKeyspaceAndTableBySource(source *Source) (*Keyspace, *Table, error) { 980 sourceKsname := source.Qualifier.String() 981 sourceTname := source.Name.String() 982 983 sourceKs, ok := vschema.Keyspaces[sourceKsname] 984 if !ok { 985 return nil, nil, vterrors.NewErrorf( 986 vtrpcpb.Code_NOT_FOUND, 987 vterrors.BadDb, 988 "source %q references a non-existent keyspace %q", 989 source, 990 sourceKsname, 991 ) 992 } 993 994 sourceT, ok := sourceKs.Tables[sourceTname] 995 if !ok { 996 return sourceKs.Keyspace, nil, vterrors.NewErrorf( 997 vtrpcpb.Code_NOT_FOUND, 998 vterrors.UnknownTable, 999 "source %q references a table %q that is not present in the VSchema of keyspace %q", source, sourceTname, sourceKsname, 1000 ) 1001 } 1002 1003 return sourceKs.Keyspace, sourceT, nil 1004 } 1005 1006 // NotFoundError represents the error where the table name was not found 1007 type NotFoundError struct { 1008 TableName string 1009 } 1010 1011 func (n NotFoundError) Error() string { 1012 return fmt.Sprintf("table %s not found", n.TableName) 1013 } 1014 1015 // FindVindex finds a vindex by name. If a keyspace is specified, only vindexes 1016 // from that keyspace are searched. If no kesypace is specified, then a vindex 1017 // is returned only if its name is unique across all keyspaces. The function 1018 // returns an error only if the vindex name is ambiguous. 1019 func (vschema *VSchema) FindVindex(keyspace, name string) (Vindex, error) { 1020 if keyspace == "" { 1021 vindex, ok := vschema.uniqueVindexes[name] 1022 if vindex == nil && ok { 1023 return nil, vterrors.Errorf( 1024 vtrpcpb.Code_FAILED_PRECONDITION, 1025 "ambiguous vindex reference: %s", 1026 name, 1027 ) 1028 } 1029 return vindex, nil 1030 } 1031 ks, ok := vschema.Keyspaces[keyspace] 1032 if !ok { 1033 return nil, vterrors.VT05003(keyspace) 1034 } 1035 return ks.Vindexes[name], nil 1036 } 1037 1038 func getShardRoutingRulesKey(keyspace, shard string) string { 1039 return fmt.Sprintf("%s.%s", keyspace, shard) 1040 } 1041 1042 // FindRoutedShard looks up shard routing rules and returns the target keyspace if applicable 1043 func (vschema *VSchema) FindRoutedShard(keyspace, shard string) (string, error) { 1044 if len(vschema.ShardRoutingRules) == 0 { 1045 return keyspace, nil 1046 } 1047 if ks, ok := vschema.ShardRoutingRules[getShardRoutingRulesKey(keyspace, shard)]; ok { 1048 return ks, nil 1049 } 1050 return keyspace, nil 1051 } 1052 1053 // ByCost provides the interface needed for ColumnVindexes to 1054 // be sorted by cost order. 1055 type ByCost []*ColumnVindex 1056 1057 func (bc ByCost) Len() int { return len(bc) } 1058 func (bc ByCost) Swap(i, j int) { bc[i], bc[j] = bc[j], bc[i] } 1059 func (bc ByCost) Less(i, j int) bool { return bc[i].Cost() < bc[j].Cost() } 1060 1061 func colVindexSorted(cvs []*ColumnVindex) (sorted []*ColumnVindex) { 1062 sorted = append(sorted, cvs...) 1063 sort.Sort(ByCost(sorted)) 1064 return sorted 1065 } 1066 1067 // LoadFormal loads the JSON representation of VSchema from a file. 1068 func LoadFormal(filename string) (*vschemapb.SrvVSchema, error) { 1069 formal := &vschemapb.SrvVSchema{} 1070 if filename == "" { 1071 return formal, nil 1072 } 1073 data, err := os.ReadFile(filename) 1074 if err != nil { 1075 return nil, err 1076 } 1077 err = json2.Unmarshal(data, formal) 1078 if err != nil { 1079 return nil, err 1080 } 1081 return formal, nil 1082 } 1083 1084 // LoadFormalKeyspace loads the JSON representation of VSchema from a file, 1085 // for a single keyspace. 1086 func LoadFormalKeyspace(filename string) (*vschemapb.Keyspace, error) { 1087 formal := &vschemapb.Keyspace{} 1088 if filename == "" { 1089 return formal, nil 1090 } 1091 data, err := os.ReadFile(filename) 1092 if err != nil { 1093 return nil, err 1094 } 1095 err = json2.Unmarshal(data, formal) 1096 if err != nil { 1097 return nil, err 1098 } 1099 return formal, nil 1100 } 1101 1102 // ChooseVindexForType chooses the most appropriate vindex for the give type. 1103 func ChooseVindexForType(typ querypb.Type) (string, error) { 1104 switch { 1105 case sqltypes.IsIntegral(typ): 1106 return "hash", nil 1107 case sqltypes.IsText(typ): 1108 return "unicode_loose_md5", nil 1109 case sqltypes.IsBinary(typ): 1110 return "binary_md5", nil 1111 } 1112 return "", vterrors.Errorf( 1113 vtrpcpb.Code_INVALID_ARGUMENT, 1114 "type %v is not recommended for a vindex", 1115 typ, 1116 ) 1117 } 1118 1119 // FindBestColVindex finds the best ColumnVindex for VReplication. 1120 func FindBestColVindex(table *Table) (*ColumnVindex, error) { 1121 if table.ColumnVindexes == nil || len(table.ColumnVindexes) == 0 { 1122 return nil, vterrors.Errorf( 1123 vtrpcpb.Code_INVALID_ARGUMENT, 1124 "table %s has no vindex", 1125 table.Name.String(), 1126 ) 1127 } 1128 var result *ColumnVindex 1129 for _, cv := range table.ColumnVindexes { 1130 if cv.Vindex.NeedsVCursor() { 1131 continue 1132 } 1133 if !cv.IsUnique() { 1134 continue 1135 } 1136 if result == nil || result.Cost() > cv.Cost() { 1137 result = cv 1138 } 1139 } 1140 if result == nil { 1141 return nil, vterrors.Errorf( 1142 vtrpcpb.Code_NOT_FOUND, 1143 "could not find a vindex to compute keyspace id for table %v", 1144 table.Name.String(), 1145 ) 1146 } 1147 return result, nil 1148 } 1149 1150 // FindVindexForSharding searches through the given slice 1151 // to find the lowest cost unique vindex 1152 // primary vindex is always unique 1153 // if two have the same cost, use the one that occurs earlier in the definition 1154 // if the final result is too expensive, return nil 1155 func FindVindexForSharding(tableName string, colVindexes []*ColumnVindex) (*ColumnVindex, error) { 1156 if len(colVindexes) == 0 { 1157 return nil, vterrors.Errorf( 1158 vtrpcpb.Code_NOT_FOUND, 1159 "no vindex definition for table %v", 1160 tableName, 1161 ) 1162 } 1163 result := colVindexes[0] 1164 for _, colVindex := range colVindexes { 1165 // Only allow SingleColumn for legacy resharding. 1166 if _, ok := colVindex.Vindex.(SingleColumn); !ok { 1167 continue 1168 } 1169 if colVindex.Cost() < result.Cost() && colVindex.IsUnique() { 1170 result = colVindex 1171 } 1172 } 1173 if result.Cost() > 1 || !result.IsUnique() { 1174 return nil, vterrors.Errorf( 1175 vtrpcpb.Code_NOT_FOUND, 1176 "could not find a vindex to use for sharding table %v", 1177 tableName, 1178 ) 1179 } 1180 return result, nil 1181 } 1182 1183 // String prints the (possibly qualified) table name 1184 func (t *Table) String() string { 1185 res := "" 1186 if t == nil { 1187 return res 1188 } 1189 if t.Keyspace != nil { 1190 res = t.Keyspace.Name + "." 1191 } 1192 return res + t.Name.String() 1193 } 1194 1195 func (t *Table) addReferenceInKeyspace(keyspace string, table *Table) { 1196 if t.ReferencedBy == nil { 1197 t.ReferencedBy = make(map[string]*Table) 1198 } 1199 t.ReferencedBy[keyspace] = table 1200 } 1201 1202 func (t *Table) getReferenceInKeyspace(keyspace string) *Table { 1203 if t.ReferencedBy == nil { 1204 return nil 1205 } 1206 t, ok := t.ReferencedBy[keyspace] 1207 if !ok { 1208 return nil 1209 } 1210 return t 1211 } 1212 1213 func (t *Table) isReferencedInKeyspace(keyspace string) bool { 1214 return t.getReferenceInKeyspace(keyspace) != nil 1215 }