vitess.io/vitess@v0.16.2/go/vt/vtgate/planbuilder/builder.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package planbuilder 18 19 import ( 20 "fmt" 21 "sort" 22 23 "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" 24 25 "vitess.io/vitess/go/sqltypes" 26 querypb "vitess.io/vitess/go/vt/proto/query" 27 28 "vitess.io/vitess/go/vt/vterrors" 29 30 "vitess.io/vitess/go/vt/key" 31 "vitess.io/vitess/go/vt/sqlparser" 32 "vitess.io/vitess/go/vt/vtgate/engine" 33 "vitess.io/vitess/go/vt/vtgate/semantics" 34 "vitess.io/vitess/go/vt/vtgate/vindexes" 35 ) 36 37 const ( 38 // V3 is also the default planner 39 V3 = querypb.ExecuteOptions_V3 40 // Gen4 uses the default Gen4 planner, which is the greedy planner 41 Gen4 = querypb.ExecuteOptions_Gen4 42 // Gen4GreedyOnly uses only the faster greedy planner 43 Gen4GreedyOnly = querypb.ExecuteOptions_Gen4Greedy 44 // Gen4Left2Right tries to emulate the V3 planner by only joining plans in the order they are listed in the FROM-clause 45 Gen4Left2Right = querypb.ExecuteOptions_Gen4Left2Right 46 // Gen4WithFallback first attempts to use the Gen4 planner, and if that fails, uses the V3 planner instead 47 Gen4WithFallback = querypb.ExecuteOptions_Gen4WithFallback 48 // Gen4CompareV3 executes queries on both Gen4 and V3 to compare their results. 49 Gen4CompareV3 = querypb.ExecuteOptions_Gen4CompareV3 50 ) 51 52 var ( 53 plannerVersions = []plancontext.PlannerVersion{V3, Gen4, Gen4GreedyOnly, Gen4Left2Right, Gen4WithFallback, Gen4CompareV3} 54 ) 55 56 type ( 57 truncater interface { 58 SetTruncateColumnCount(int) 59 } 60 61 planResult struct { 62 primitive engine.Primitive 63 tables []string 64 } 65 66 stmtPlanner func(sqlparser.Statement, *sqlparser.ReservedVars, plancontext.VSchema) (*planResult, error) 67 ) 68 69 func newPlanResult(prim engine.Primitive, tablesUsed ...string) *planResult { 70 return &planResult{primitive: prim, tables: tablesUsed} 71 } 72 73 func singleTable(ks, tbl string) string { 74 return fmt.Sprintf("%s.%s", ks, tbl) 75 } 76 77 func tablesFromSemantics(semTable *semantics.SemTable) []string { 78 tables := make(map[string]any, len(semTable.Tables)) 79 for _, info := range semTable.Tables { 80 vindexTable := info.GetVindexTable() 81 if vindexTable == nil { 82 continue 83 } 84 tables[vindexTable.String()] = nil 85 } 86 87 names := make([]string, 0, len(tables)) 88 for tbl := range tables { 89 names = append(names, tbl) 90 } 91 sort.Strings(names) 92 return names 93 } 94 95 // TestBuilder builds a plan for a query based on the specified vschema. 96 // This method is only used from tests 97 func TestBuilder(query string, vschema plancontext.VSchema, keyspace string) (*engine.Plan, error) { 98 stmt, reserved, err := sqlparser.Parse2(query) 99 if err != nil { 100 return nil, err 101 } 102 result, err := sqlparser.RewriteAST(stmt, keyspace, sqlparser.SQLSelectLimitUnset, "", nil, vschema) 103 if err != nil { 104 return nil, err 105 } 106 107 reservedVars := sqlparser.NewReservedVars("vtg", reserved) 108 return BuildFromStmt(query, result.AST, reservedVars, vschema, result.BindVarNeeds, true, true) 109 } 110 111 // BuildFromStmt builds a plan based on the AST provided. 112 func BuildFromStmt(query string, stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, bindVarNeeds *sqlparser.BindVarNeeds, enableOnlineDDL, enableDirectDDL bool) (*engine.Plan, error) { 113 planResult, err := createInstructionFor(query, stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL) 114 if err != nil { 115 return nil, err 116 } 117 118 var primitive engine.Primitive 119 var tablesUsed []string 120 if planResult != nil { 121 primitive = planResult.primitive 122 tablesUsed = planResult.tables 123 } 124 plan := &engine.Plan{ 125 Type: sqlparser.ASTToStatementType(stmt), 126 Original: query, 127 Instructions: primitive, 128 BindVarNeeds: bindVarNeeds, 129 TablesUsed: tablesUsed, 130 } 131 return plan, nil 132 } 133 134 func getConfiguredPlanner(vschema plancontext.VSchema, v3planner func(string) stmtPlanner, stmt sqlparser.Statement, query string) (stmtPlanner, error) { 135 planner, ok := getPlannerFromQuery(stmt) 136 if !ok { 137 // if the query doesn't specify the planner, we check what the configuration is 138 planner = vschema.Planner() 139 } 140 switch planner { 141 case Gen4CompareV3: 142 return gen4CompareV3Planner(query), nil 143 case Gen4Left2Right, Gen4GreedyOnly: 144 return gen4Planner(query, planner), nil 145 case Gen4WithFallback: 146 fp := &fallbackPlanner{ 147 primary: gen4Planner(query, querypb.ExecuteOptions_Gen4), 148 fallback: v3planner(query), 149 } 150 return fp.plan, nil 151 case V3: 152 return v3planner(query), nil 153 default: 154 // default is gen4 plan 155 return gen4Planner(query, Gen4), nil 156 } 157 } 158 159 // getPlannerFromQuery chooses the planner to use based on the query 160 // The default planner can be overridden using /*vt+ PLANNER=gen4 */ 161 // We will also fall back on the gen4 planner if we encounter outer join, 162 // since there are known problems with the v3 planner and outer joins 163 func getPlannerFromQuery(stmt sqlparser.Statement) (version plancontext.PlannerVersion, found bool) { 164 version, found = getPlannerFromQueryHint(stmt) 165 if found { 166 return 167 } 168 169 _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { 170 join, ok := node.(*sqlparser.JoinTableExpr) 171 if ok { 172 if join.Join == sqlparser.LeftJoinType || join.Join == sqlparser.RightJoinType { 173 version = querypb.ExecuteOptions_Gen4 174 found = true 175 return false, nil 176 } 177 } 178 return true, nil 179 }, stmt) 180 181 return 182 } 183 184 func getPlannerFromQueryHint(stmt sqlparser.Statement) (plancontext.PlannerVersion, bool) { 185 cm, isCom := stmt.(sqlparser.Commented) 186 if !isCom { 187 return plancontext.PlannerVersion(0), false 188 } 189 190 d := cm.GetParsedComments().Directives() 191 val, ok := d.GetString(sqlparser.DirectiveQueryPlanner, "") 192 if !ok { 193 return plancontext.PlannerVersion(0), false 194 } 195 return plancontext.PlannerNameToVersion(val) 196 } 197 198 func buildRoutePlan(stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, f func(statement sqlparser.Statement, reservedVars *sqlparser.ReservedVars, schema plancontext.VSchema) (*planResult, error)) (*planResult, error) { 199 if vschema.Destination() != nil { 200 return buildPlanForBypass(stmt, reservedVars, vschema) 201 } 202 return f(stmt, reservedVars, vschema) 203 } 204 205 func createInstructionFor(query string, stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, enableOnlineDDL, enableDirectDDL bool) (*planResult, error) { 206 switch stmt := stmt.(type) { 207 case *sqlparser.Select: 208 configuredPlanner, err := getConfiguredPlanner(vschema, buildSelectPlan, stmt, query) 209 if err != nil { 210 return nil, err 211 } 212 return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner) 213 case *sqlparser.Insert: 214 return buildRoutePlan(stmt, reservedVars, vschema, buildInsertPlan) 215 case *sqlparser.Update: 216 configuredPlanner, err := getConfiguredPlanner(vschema, buildUpdatePlan, stmt, query) 217 if err != nil { 218 return nil, err 219 } 220 return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner) 221 case *sqlparser.Delete: 222 configuredPlanner, err := getConfiguredPlanner(vschema, buildDeletePlan, stmt, query) 223 if err != nil { 224 return nil, err 225 } 226 return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner) 227 case *sqlparser.Union: 228 configuredPlanner, err := getConfiguredPlanner(vschema, buildUnionPlan, stmt, query) 229 if err != nil { 230 return nil, err 231 } 232 return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner) 233 case sqlparser.DDLStatement: 234 return buildGeneralDDLPlan(query, stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL) 235 case *sqlparser.AlterMigration: 236 return buildAlterMigrationPlan(query, vschema, enableOnlineDDL) 237 case *sqlparser.RevertMigration: 238 return buildRevertMigrationPlan(query, stmt, vschema, enableOnlineDDL) 239 case *sqlparser.ShowMigrationLogs: 240 return buildShowMigrationLogsPlan(query, vschema, enableOnlineDDL) 241 case *sqlparser.ShowThrottledApps: 242 return buildShowThrottledAppsPlan(query, vschema) 243 case *sqlparser.ShowThrottlerStatus: 244 return buildShowThrottlerStatusPlan(query, vschema) 245 case *sqlparser.AlterVschema: 246 return buildVSchemaDDLPlan(stmt, vschema) 247 case *sqlparser.Use: 248 return buildUsePlan(stmt) 249 case sqlparser.Explain: 250 return buildExplainPlan(stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL) 251 case *sqlparser.VExplainStmt: 252 return buildVExplainPlan(stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL) 253 case *sqlparser.OtherRead, *sqlparser.OtherAdmin: 254 return buildOtherReadAndAdmin(query, vschema) 255 case *sqlparser.Set: 256 return buildSetPlan(stmt, vschema) 257 case *sqlparser.Load: 258 return buildLoadPlan(query, vschema) 259 case sqlparser.DBDDLStatement: 260 return buildRoutePlan(stmt, reservedVars, vschema, buildDBDDLPlan) 261 case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback, *sqlparser.Savepoint, *sqlparser.SRollback, *sqlparser.Release: 262 // Empty by design. Not executed by a plan 263 return nil, nil 264 case *sqlparser.Show: 265 return buildShowPlan(query, stmt, reservedVars, vschema) 266 case *sqlparser.LockTables: 267 return buildRoutePlan(stmt, reservedVars, vschema, buildLockPlan) 268 case *sqlparser.UnlockTables: 269 return buildRoutePlan(stmt, reservedVars, vschema, buildUnlockPlan) 270 case *sqlparser.Flush: 271 return buildFlushPlan(stmt, vschema) 272 case *sqlparser.CallProc: 273 return buildCallProcPlan(stmt, vschema) 274 case *sqlparser.Stream: 275 return buildStreamPlan(stmt, vschema) 276 case *sqlparser.VStream: 277 return buildVStreamPlan(stmt, vschema) 278 case *sqlparser.CommentOnly: 279 // There is only a comment in the input. 280 // This is essentially a No-op 281 return newPlanResult(engine.NewRowsPrimitive(nil, nil)), nil 282 } 283 284 return nil, vterrors.VT13001(fmt.Sprintf("unexpected statement type: %T", stmt)) 285 } 286 287 func buildDBDDLPlan(stmt sqlparser.Statement, _ *sqlparser.ReservedVars, vschema plancontext.VSchema) (*planResult, error) { 288 dbDDLstmt := stmt.(sqlparser.DBDDLStatement) 289 ksName := dbDDLstmt.GetDatabaseName() 290 if ksName == "" { 291 ks, err := vschema.DefaultKeyspace() 292 if err != nil { 293 return nil, err 294 } 295 ksName = ks.Name 296 } 297 ksExists := vschema.KeyspaceExists(ksName) 298 299 switch dbDDL := dbDDLstmt.(type) { 300 case *sqlparser.DropDatabase: 301 if dbDDL.IfExists && !ksExists { 302 return newPlanResult(engine.NewRowsPrimitive(make([][]sqltypes.Value, 0), make([]*querypb.Field, 0))), nil 303 } 304 if !ksExists { 305 return nil, vterrors.VT05001(ksName) 306 } 307 return newPlanResult(engine.NewDBDDL(ksName, false, queryTimeout(dbDDL.Comments.Directives()))), nil 308 case *sqlparser.AlterDatabase: 309 if !ksExists { 310 return nil, vterrors.VT05002(ksName) 311 } 312 return nil, vterrors.VT12001("ALTER DATABASE") 313 case *sqlparser.CreateDatabase: 314 if dbDDL.IfNotExists && ksExists { 315 return newPlanResult(engine.NewRowsPrimitive(make([][]sqltypes.Value, 0), make([]*querypb.Field, 0))), nil 316 } 317 if !dbDDL.IfNotExists && ksExists { 318 return nil, vterrors.VT06001(ksName) 319 } 320 return newPlanResult(engine.NewDBDDL(ksName, true, queryTimeout(dbDDL.Comments.Directives()))), nil 321 } 322 return nil, vterrors.VT13001(fmt.Sprintf("database DDL not recognized: %s", sqlparser.String(dbDDLstmt))) 323 } 324 325 func buildLoadPlan(query string, vschema plancontext.VSchema) (*planResult, error) { 326 keyspace, err := vschema.DefaultKeyspace() 327 if err != nil { 328 return nil, err 329 } 330 331 destination := vschema.Destination() 332 if destination == nil { 333 if err := vschema.ErrorIfShardedF(keyspace, "LOAD", "LOAD is not supported on sharded keyspace"); err != nil { 334 return nil, err 335 } 336 destination = key.DestinationAnyShard{} 337 } 338 339 return newPlanResult(&engine.Send{ 340 Keyspace: keyspace, 341 TargetDestination: destination, 342 Query: query, 343 IsDML: true, 344 SingleShardOnly: true, 345 }), nil 346 } 347 348 func buildVSchemaDDLPlan(stmt *sqlparser.AlterVschema, vschema plancontext.VSchema) (*planResult, error) { 349 _, keyspace, _, err := vschema.TargetDestination(stmt.Table.Qualifier.String()) 350 if err != nil { 351 return nil, err 352 } 353 return newPlanResult(&engine.AlterVSchema{ 354 Keyspace: keyspace, 355 AlterVschemaDDL: stmt, 356 }, singleTable(keyspace.Name, stmt.Table.Name.String())), nil 357 } 358 359 func buildFlushPlan(stmt *sqlparser.Flush, vschema plancontext.VSchema) (*planResult, error) { 360 if len(stmt.TableNames) == 0 { 361 return buildFlushOptions(stmt, vschema) 362 } 363 return buildFlushTables(stmt, vschema) 364 } 365 366 func buildFlushOptions(stmt *sqlparser.Flush, vschema plancontext.VSchema) (*planResult, error) { 367 dest, keyspace, _, err := vschema.TargetDestination("") 368 if err != nil { 369 return nil, err 370 } 371 if dest == nil { 372 dest = key.DestinationAllShards{} 373 } 374 tc := &tableCollector{} 375 for _, tbl := range stmt.TableNames { 376 tc.addASTTable(keyspace.Name, tbl) 377 } 378 379 return newPlanResult(&engine.Send{ 380 Keyspace: keyspace, 381 TargetDestination: dest, 382 Query: sqlparser.String(stmt), 383 IsDML: false, 384 SingleShardOnly: false, 385 }, tc.getTables()...), nil 386 } 387 388 func buildFlushTables(stmt *sqlparser.Flush, vschema plancontext.VSchema) (*planResult, error) { 389 tc := &tableCollector{} 390 type sendDest struct { 391 ks *vindexes.Keyspace 392 dest key.Destination 393 } 394 395 dest := vschema.Destination() 396 if dest == nil { 397 dest = key.DestinationAllShards{} 398 } 399 400 tablesMap := make(map[sendDest]sqlparser.TableNames) 401 var keys []sendDest 402 for i, tab := range stmt.TableNames { 403 var ksTab *vindexes.Keyspace 404 var table *vindexes.Table 405 var err error 406 407 table, _, _, _, _, err = vschema.FindTableOrVindex(tab) 408 if err != nil { 409 return nil, err 410 } 411 if table == nil { 412 return nil, vindexes.NotFoundError{TableName: tab.Name.String()} 413 } 414 tc.addTable(table.Keyspace.Name, table.Name.String()) 415 ksTab = table.Keyspace 416 stmt.TableNames[i] = sqlparser.TableName{ 417 Name: table.Name, 418 } 419 420 key := sendDest{ksTab, dest} 421 tables, isAvail := tablesMap[key] 422 if !isAvail { 423 keys = append(keys, key) 424 } 425 tables = append(tables, stmt.TableNames[i]) 426 tablesMap[key] = tables 427 } 428 429 if len(tablesMap) == 1 { 430 for sendDest, tables := range tablesMap { 431 return newPlanResult(&engine.Send{ 432 Keyspace: sendDest.ks, 433 TargetDestination: sendDest.dest, 434 Query: sqlparser.String(newFlushStmt(stmt, tables)), 435 }, tc.getTables()...), nil 436 } 437 } 438 439 sort.Slice(keys, func(i, j int) bool { 440 return keys[i].ks.Name < keys[j].ks.Name 441 }) 442 443 var sources []engine.Primitive 444 for _, sendDest := range keys { 445 plan := &engine.Send{ 446 Keyspace: sendDest.ks, 447 TargetDestination: sendDest.dest, 448 Query: sqlparser.String(newFlushStmt(stmt, tablesMap[sendDest])), 449 } 450 sources = append(sources, plan) 451 } 452 return newPlanResult(engine.NewConcatenate(sources, nil), tc.getTables()...), nil 453 } 454 455 type tableCollector struct { 456 tables map[string]any 457 } 458 459 func (tc *tableCollector) addTable(ks, tbl string) { 460 if tc.tables == nil { 461 tc.tables = map[string]any{} 462 } 463 tc.tables[fmt.Sprintf("%s.%s", ks, tbl)] = nil 464 } 465 466 func (tc *tableCollector) addASTTable(ks string, tbl sqlparser.TableName) { 467 tc.addTable(ks, tbl.Name.String()) 468 } 469 470 func (tc *tableCollector) getTables() []string { 471 tableNames := make([]string, 0, len(tc.tables)) 472 for tbl := range tc.tables { 473 tableNames = append(tableNames, tbl) 474 } 475 476 sort.Strings(tableNames) 477 return tableNames 478 } 479 480 func (tc *tableCollector) addVindexTable(t *vindexes.Table) { 481 if t == nil { 482 return 483 } 484 ks, tbl := "", t.Name.String() 485 if t.Keyspace != nil { 486 ks = t.Keyspace.Name 487 } 488 tc.addTable(ks, tbl) 489 } 490 491 func (tc *tableCollector) addAllTables(tables []string) { 492 if tc.tables == nil { 493 tc.tables = map[string]any{} 494 } 495 for _, tbl := range tables { 496 tc.tables[tbl] = nil 497 } 498 } 499 500 func newFlushStmt(stmt *sqlparser.Flush, tables sqlparser.TableNames) *sqlparser.Flush { 501 return &sqlparser.Flush{ 502 IsLocal: stmt.IsLocal, 503 TableNames: tables, 504 WithLock: stmt.WithLock, 505 ForExport: stmt.ForExport, 506 } 507 }