vitess.io/vitess@v0.16.2/go/vt/vtgate/planbuilder/builder.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package planbuilder
    18  
    19  import (
    20  	"fmt"
    21  	"sort"
    22  
    23  	"vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext"
    24  
    25  	"vitess.io/vitess/go/sqltypes"
    26  	querypb "vitess.io/vitess/go/vt/proto/query"
    27  
    28  	"vitess.io/vitess/go/vt/vterrors"
    29  
    30  	"vitess.io/vitess/go/vt/key"
    31  	"vitess.io/vitess/go/vt/sqlparser"
    32  	"vitess.io/vitess/go/vt/vtgate/engine"
    33  	"vitess.io/vitess/go/vt/vtgate/semantics"
    34  	"vitess.io/vitess/go/vt/vtgate/vindexes"
    35  )
    36  
    37  const (
    38  	// V3 is also the default planner
    39  	V3 = querypb.ExecuteOptions_V3
    40  	// Gen4 uses the default Gen4 planner, which is the greedy planner
    41  	Gen4 = querypb.ExecuteOptions_Gen4
    42  	// Gen4GreedyOnly uses only the faster greedy planner
    43  	Gen4GreedyOnly = querypb.ExecuteOptions_Gen4Greedy
    44  	// Gen4Left2Right tries to emulate the V3 planner by only joining plans in the order they are listed in the FROM-clause
    45  	Gen4Left2Right = querypb.ExecuteOptions_Gen4Left2Right
    46  	// Gen4WithFallback first attempts to use the Gen4 planner, and if that fails, uses the V3 planner instead
    47  	Gen4WithFallback = querypb.ExecuteOptions_Gen4WithFallback
    48  	// Gen4CompareV3 executes queries on both Gen4 and V3 to compare their results.
    49  	Gen4CompareV3 = querypb.ExecuteOptions_Gen4CompareV3
    50  )
    51  
    52  var (
    53  	plannerVersions = []plancontext.PlannerVersion{V3, Gen4, Gen4GreedyOnly, Gen4Left2Right, Gen4WithFallback, Gen4CompareV3}
    54  )
    55  
    56  type (
    57  	truncater interface {
    58  		SetTruncateColumnCount(int)
    59  	}
    60  
    61  	planResult struct {
    62  		primitive engine.Primitive
    63  		tables    []string
    64  	}
    65  
    66  	stmtPlanner func(sqlparser.Statement, *sqlparser.ReservedVars, plancontext.VSchema) (*planResult, error)
    67  )
    68  
    69  func newPlanResult(prim engine.Primitive, tablesUsed ...string) *planResult {
    70  	return &planResult{primitive: prim, tables: tablesUsed}
    71  }
    72  
    73  func singleTable(ks, tbl string) string {
    74  	return fmt.Sprintf("%s.%s", ks, tbl)
    75  }
    76  
    77  func tablesFromSemantics(semTable *semantics.SemTable) []string {
    78  	tables := make(map[string]any, len(semTable.Tables))
    79  	for _, info := range semTable.Tables {
    80  		vindexTable := info.GetVindexTable()
    81  		if vindexTable == nil {
    82  			continue
    83  		}
    84  		tables[vindexTable.String()] = nil
    85  	}
    86  
    87  	names := make([]string, 0, len(tables))
    88  	for tbl := range tables {
    89  		names = append(names, tbl)
    90  	}
    91  	sort.Strings(names)
    92  	return names
    93  }
    94  
    95  // TestBuilder builds a plan for a query based on the specified vschema.
    96  // This method is only used from tests
    97  func TestBuilder(query string, vschema plancontext.VSchema, keyspace string) (*engine.Plan, error) {
    98  	stmt, reserved, err := sqlparser.Parse2(query)
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  	result, err := sqlparser.RewriteAST(stmt, keyspace, sqlparser.SQLSelectLimitUnset, "", nil, vschema)
   103  	if err != nil {
   104  		return nil, err
   105  	}
   106  
   107  	reservedVars := sqlparser.NewReservedVars("vtg", reserved)
   108  	return BuildFromStmt(query, result.AST, reservedVars, vschema, result.BindVarNeeds, true, true)
   109  }
   110  
   111  // BuildFromStmt builds a plan based on the AST provided.
   112  func BuildFromStmt(query string, stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, bindVarNeeds *sqlparser.BindVarNeeds, enableOnlineDDL, enableDirectDDL bool) (*engine.Plan, error) {
   113  	planResult, err := createInstructionFor(query, stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL)
   114  	if err != nil {
   115  		return nil, err
   116  	}
   117  
   118  	var primitive engine.Primitive
   119  	var tablesUsed []string
   120  	if planResult != nil {
   121  		primitive = planResult.primitive
   122  		tablesUsed = planResult.tables
   123  	}
   124  	plan := &engine.Plan{
   125  		Type:         sqlparser.ASTToStatementType(stmt),
   126  		Original:     query,
   127  		Instructions: primitive,
   128  		BindVarNeeds: bindVarNeeds,
   129  		TablesUsed:   tablesUsed,
   130  	}
   131  	return plan, nil
   132  }
   133  
   134  func getConfiguredPlanner(vschema plancontext.VSchema, v3planner func(string) stmtPlanner, stmt sqlparser.Statement, query string) (stmtPlanner, error) {
   135  	planner, ok := getPlannerFromQuery(stmt)
   136  	if !ok {
   137  		// if the query doesn't specify the planner, we check what the configuration is
   138  		planner = vschema.Planner()
   139  	}
   140  	switch planner {
   141  	case Gen4CompareV3:
   142  		return gen4CompareV3Planner(query), nil
   143  	case Gen4Left2Right, Gen4GreedyOnly:
   144  		return gen4Planner(query, planner), nil
   145  	case Gen4WithFallback:
   146  		fp := &fallbackPlanner{
   147  			primary:  gen4Planner(query, querypb.ExecuteOptions_Gen4),
   148  			fallback: v3planner(query),
   149  		}
   150  		return fp.plan, nil
   151  	case V3:
   152  		return v3planner(query), nil
   153  	default:
   154  		// default is gen4 plan
   155  		return gen4Planner(query, Gen4), nil
   156  	}
   157  }
   158  
   159  // getPlannerFromQuery chooses the planner to use based on the query
   160  // The default planner can be overridden using /*vt+ PLANNER=gen4 */
   161  // We will also fall back on the gen4 planner if we encounter outer join,
   162  // since there are known problems with the v3 planner and outer joins
   163  func getPlannerFromQuery(stmt sqlparser.Statement) (version plancontext.PlannerVersion, found bool) {
   164  	version, found = getPlannerFromQueryHint(stmt)
   165  	if found {
   166  		return
   167  	}
   168  
   169  	_ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) {
   170  		join, ok := node.(*sqlparser.JoinTableExpr)
   171  		if ok {
   172  			if join.Join == sqlparser.LeftJoinType || join.Join == sqlparser.RightJoinType {
   173  				version = querypb.ExecuteOptions_Gen4
   174  				found = true
   175  				return false, nil
   176  			}
   177  		}
   178  		return true, nil
   179  	}, stmt)
   180  
   181  	return
   182  }
   183  
   184  func getPlannerFromQueryHint(stmt sqlparser.Statement) (plancontext.PlannerVersion, bool) {
   185  	cm, isCom := stmt.(sqlparser.Commented)
   186  	if !isCom {
   187  		return plancontext.PlannerVersion(0), false
   188  	}
   189  
   190  	d := cm.GetParsedComments().Directives()
   191  	val, ok := d.GetString(sqlparser.DirectiveQueryPlanner, "")
   192  	if !ok {
   193  		return plancontext.PlannerVersion(0), false
   194  	}
   195  	return plancontext.PlannerNameToVersion(val)
   196  }
   197  
   198  func buildRoutePlan(stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, f func(statement sqlparser.Statement, reservedVars *sqlparser.ReservedVars, schema plancontext.VSchema) (*planResult, error)) (*planResult, error) {
   199  	if vschema.Destination() != nil {
   200  		return buildPlanForBypass(stmt, reservedVars, vschema)
   201  	}
   202  	return f(stmt, reservedVars, vschema)
   203  }
   204  
   205  func createInstructionFor(query string, stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, enableOnlineDDL, enableDirectDDL bool) (*planResult, error) {
   206  	switch stmt := stmt.(type) {
   207  	case *sqlparser.Select:
   208  		configuredPlanner, err := getConfiguredPlanner(vschema, buildSelectPlan, stmt, query)
   209  		if err != nil {
   210  			return nil, err
   211  		}
   212  		return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner)
   213  	case *sqlparser.Insert:
   214  		return buildRoutePlan(stmt, reservedVars, vschema, buildInsertPlan)
   215  	case *sqlparser.Update:
   216  		configuredPlanner, err := getConfiguredPlanner(vschema, buildUpdatePlan, stmt, query)
   217  		if err != nil {
   218  			return nil, err
   219  		}
   220  		return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner)
   221  	case *sqlparser.Delete:
   222  		configuredPlanner, err := getConfiguredPlanner(vschema, buildDeletePlan, stmt, query)
   223  		if err != nil {
   224  			return nil, err
   225  		}
   226  		return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner)
   227  	case *sqlparser.Union:
   228  		configuredPlanner, err := getConfiguredPlanner(vschema, buildUnionPlan, stmt, query)
   229  		if err != nil {
   230  			return nil, err
   231  		}
   232  		return buildRoutePlan(stmt, reservedVars, vschema, configuredPlanner)
   233  	case sqlparser.DDLStatement:
   234  		return buildGeneralDDLPlan(query, stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL)
   235  	case *sqlparser.AlterMigration:
   236  		return buildAlterMigrationPlan(query, vschema, enableOnlineDDL)
   237  	case *sqlparser.RevertMigration:
   238  		return buildRevertMigrationPlan(query, stmt, vschema, enableOnlineDDL)
   239  	case *sqlparser.ShowMigrationLogs:
   240  		return buildShowMigrationLogsPlan(query, vschema, enableOnlineDDL)
   241  	case *sqlparser.ShowThrottledApps:
   242  		return buildShowThrottledAppsPlan(query, vschema)
   243  	case *sqlparser.ShowThrottlerStatus:
   244  		return buildShowThrottlerStatusPlan(query, vschema)
   245  	case *sqlparser.AlterVschema:
   246  		return buildVSchemaDDLPlan(stmt, vschema)
   247  	case *sqlparser.Use:
   248  		return buildUsePlan(stmt)
   249  	case sqlparser.Explain:
   250  		return buildExplainPlan(stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL)
   251  	case *sqlparser.VExplainStmt:
   252  		return buildVExplainPlan(stmt, reservedVars, vschema, enableOnlineDDL, enableDirectDDL)
   253  	case *sqlparser.OtherRead, *sqlparser.OtherAdmin:
   254  		return buildOtherReadAndAdmin(query, vschema)
   255  	case *sqlparser.Set:
   256  		return buildSetPlan(stmt, vschema)
   257  	case *sqlparser.Load:
   258  		return buildLoadPlan(query, vschema)
   259  	case sqlparser.DBDDLStatement:
   260  		return buildRoutePlan(stmt, reservedVars, vschema, buildDBDDLPlan)
   261  	case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback, *sqlparser.Savepoint, *sqlparser.SRollback, *sqlparser.Release:
   262  		// Empty by design. Not executed by a plan
   263  		return nil, nil
   264  	case *sqlparser.Show:
   265  		return buildShowPlan(query, stmt, reservedVars, vschema)
   266  	case *sqlparser.LockTables:
   267  		return buildRoutePlan(stmt, reservedVars, vschema, buildLockPlan)
   268  	case *sqlparser.UnlockTables:
   269  		return buildRoutePlan(stmt, reservedVars, vschema, buildUnlockPlan)
   270  	case *sqlparser.Flush:
   271  		return buildFlushPlan(stmt, vschema)
   272  	case *sqlparser.CallProc:
   273  		return buildCallProcPlan(stmt, vschema)
   274  	case *sqlparser.Stream:
   275  		return buildStreamPlan(stmt, vschema)
   276  	case *sqlparser.VStream:
   277  		return buildVStreamPlan(stmt, vschema)
   278  	case *sqlparser.CommentOnly:
   279  		// There is only a comment in the input.
   280  		// This is essentially a No-op
   281  		return newPlanResult(engine.NewRowsPrimitive(nil, nil)), nil
   282  	}
   283  
   284  	return nil, vterrors.VT13001(fmt.Sprintf("unexpected statement type: %T", stmt))
   285  }
   286  
   287  func buildDBDDLPlan(stmt sqlparser.Statement, _ *sqlparser.ReservedVars, vschema plancontext.VSchema) (*planResult, error) {
   288  	dbDDLstmt := stmt.(sqlparser.DBDDLStatement)
   289  	ksName := dbDDLstmt.GetDatabaseName()
   290  	if ksName == "" {
   291  		ks, err := vschema.DefaultKeyspace()
   292  		if err != nil {
   293  			return nil, err
   294  		}
   295  		ksName = ks.Name
   296  	}
   297  	ksExists := vschema.KeyspaceExists(ksName)
   298  
   299  	switch dbDDL := dbDDLstmt.(type) {
   300  	case *sqlparser.DropDatabase:
   301  		if dbDDL.IfExists && !ksExists {
   302  			return newPlanResult(engine.NewRowsPrimitive(make([][]sqltypes.Value, 0), make([]*querypb.Field, 0))), nil
   303  		}
   304  		if !ksExists {
   305  			return nil, vterrors.VT05001(ksName)
   306  		}
   307  		return newPlanResult(engine.NewDBDDL(ksName, false, queryTimeout(dbDDL.Comments.Directives()))), nil
   308  	case *sqlparser.AlterDatabase:
   309  		if !ksExists {
   310  			return nil, vterrors.VT05002(ksName)
   311  		}
   312  		return nil, vterrors.VT12001("ALTER DATABASE")
   313  	case *sqlparser.CreateDatabase:
   314  		if dbDDL.IfNotExists && ksExists {
   315  			return newPlanResult(engine.NewRowsPrimitive(make([][]sqltypes.Value, 0), make([]*querypb.Field, 0))), nil
   316  		}
   317  		if !dbDDL.IfNotExists && ksExists {
   318  			return nil, vterrors.VT06001(ksName)
   319  		}
   320  		return newPlanResult(engine.NewDBDDL(ksName, true, queryTimeout(dbDDL.Comments.Directives()))), nil
   321  	}
   322  	return nil, vterrors.VT13001(fmt.Sprintf("database DDL not recognized: %s", sqlparser.String(dbDDLstmt)))
   323  }
   324  
   325  func buildLoadPlan(query string, vschema plancontext.VSchema) (*planResult, error) {
   326  	keyspace, err := vschema.DefaultKeyspace()
   327  	if err != nil {
   328  		return nil, err
   329  	}
   330  
   331  	destination := vschema.Destination()
   332  	if destination == nil {
   333  		if err := vschema.ErrorIfShardedF(keyspace, "LOAD", "LOAD is not supported on sharded keyspace"); err != nil {
   334  			return nil, err
   335  		}
   336  		destination = key.DestinationAnyShard{}
   337  	}
   338  
   339  	return newPlanResult(&engine.Send{
   340  		Keyspace:          keyspace,
   341  		TargetDestination: destination,
   342  		Query:             query,
   343  		IsDML:             true,
   344  		SingleShardOnly:   true,
   345  	}), nil
   346  }
   347  
   348  func buildVSchemaDDLPlan(stmt *sqlparser.AlterVschema, vschema plancontext.VSchema) (*planResult, error) {
   349  	_, keyspace, _, err := vschema.TargetDestination(stmt.Table.Qualifier.String())
   350  	if err != nil {
   351  		return nil, err
   352  	}
   353  	return newPlanResult(&engine.AlterVSchema{
   354  		Keyspace:        keyspace,
   355  		AlterVschemaDDL: stmt,
   356  	}, singleTable(keyspace.Name, stmt.Table.Name.String())), nil
   357  }
   358  
   359  func buildFlushPlan(stmt *sqlparser.Flush, vschema plancontext.VSchema) (*planResult, error) {
   360  	if len(stmt.TableNames) == 0 {
   361  		return buildFlushOptions(stmt, vschema)
   362  	}
   363  	return buildFlushTables(stmt, vschema)
   364  }
   365  
   366  func buildFlushOptions(stmt *sqlparser.Flush, vschema plancontext.VSchema) (*planResult, error) {
   367  	dest, keyspace, _, err := vschema.TargetDestination("")
   368  	if err != nil {
   369  		return nil, err
   370  	}
   371  	if dest == nil {
   372  		dest = key.DestinationAllShards{}
   373  	}
   374  	tc := &tableCollector{}
   375  	for _, tbl := range stmt.TableNames {
   376  		tc.addASTTable(keyspace.Name, tbl)
   377  	}
   378  
   379  	return newPlanResult(&engine.Send{
   380  		Keyspace:          keyspace,
   381  		TargetDestination: dest,
   382  		Query:             sqlparser.String(stmt),
   383  		IsDML:             false,
   384  		SingleShardOnly:   false,
   385  	}, tc.getTables()...), nil
   386  }
   387  
   388  func buildFlushTables(stmt *sqlparser.Flush, vschema plancontext.VSchema) (*planResult, error) {
   389  	tc := &tableCollector{}
   390  	type sendDest struct {
   391  		ks   *vindexes.Keyspace
   392  		dest key.Destination
   393  	}
   394  
   395  	dest := vschema.Destination()
   396  	if dest == nil {
   397  		dest = key.DestinationAllShards{}
   398  	}
   399  
   400  	tablesMap := make(map[sendDest]sqlparser.TableNames)
   401  	var keys []sendDest
   402  	for i, tab := range stmt.TableNames {
   403  		var ksTab *vindexes.Keyspace
   404  		var table *vindexes.Table
   405  		var err error
   406  
   407  		table, _, _, _, _, err = vschema.FindTableOrVindex(tab)
   408  		if err != nil {
   409  			return nil, err
   410  		}
   411  		if table == nil {
   412  			return nil, vindexes.NotFoundError{TableName: tab.Name.String()}
   413  		}
   414  		tc.addTable(table.Keyspace.Name, table.Name.String())
   415  		ksTab = table.Keyspace
   416  		stmt.TableNames[i] = sqlparser.TableName{
   417  			Name: table.Name,
   418  		}
   419  
   420  		key := sendDest{ksTab, dest}
   421  		tables, isAvail := tablesMap[key]
   422  		if !isAvail {
   423  			keys = append(keys, key)
   424  		}
   425  		tables = append(tables, stmt.TableNames[i])
   426  		tablesMap[key] = tables
   427  	}
   428  
   429  	if len(tablesMap) == 1 {
   430  		for sendDest, tables := range tablesMap {
   431  			return newPlanResult(&engine.Send{
   432  				Keyspace:          sendDest.ks,
   433  				TargetDestination: sendDest.dest,
   434  				Query:             sqlparser.String(newFlushStmt(stmt, tables)),
   435  			}, tc.getTables()...), nil
   436  		}
   437  	}
   438  
   439  	sort.Slice(keys, func(i, j int) bool {
   440  		return keys[i].ks.Name < keys[j].ks.Name
   441  	})
   442  
   443  	var sources []engine.Primitive
   444  	for _, sendDest := range keys {
   445  		plan := &engine.Send{
   446  			Keyspace:          sendDest.ks,
   447  			TargetDestination: sendDest.dest,
   448  			Query:             sqlparser.String(newFlushStmt(stmt, tablesMap[sendDest])),
   449  		}
   450  		sources = append(sources, plan)
   451  	}
   452  	return newPlanResult(engine.NewConcatenate(sources, nil), tc.getTables()...), nil
   453  }
   454  
   455  type tableCollector struct {
   456  	tables map[string]any
   457  }
   458  
   459  func (tc *tableCollector) addTable(ks, tbl string) {
   460  	if tc.tables == nil {
   461  		tc.tables = map[string]any{}
   462  	}
   463  	tc.tables[fmt.Sprintf("%s.%s", ks, tbl)] = nil
   464  }
   465  
   466  func (tc *tableCollector) addASTTable(ks string, tbl sqlparser.TableName) {
   467  	tc.addTable(ks, tbl.Name.String())
   468  }
   469  
   470  func (tc *tableCollector) getTables() []string {
   471  	tableNames := make([]string, 0, len(tc.tables))
   472  	for tbl := range tc.tables {
   473  		tableNames = append(tableNames, tbl)
   474  	}
   475  
   476  	sort.Strings(tableNames)
   477  	return tableNames
   478  }
   479  
   480  func (tc *tableCollector) addVindexTable(t *vindexes.Table) {
   481  	if t == nil {
   482  		return
   483  	}
   484  	ks, tbl := "", t.Name.String()
   485  	if t.Keyspace != nil {
   486  		ks = t.Keyspace.Name
   487  	}
   488  	tc.addTable(ks, tbl)
   489  }
   490  
   491  func (tc *tableCollector) addAllTables(tables []string) {
   492  	if tc.tables == nil {
   493  		tc.tables = map[string]any{}
   494  	}
   495  	for _, tbl := range tables {
   496  		tc.tables[tbl] = nil
   497  	}
   498  }
   499  
   500  func newFlushStmt(stmt *sqlparser.Flush, tables sqlparser.TableNames) *sqlparser.Flush {
   501  	return &sqlparser.Flush{
   502  		IsLocal:    stmt.IsLocal,
   503  		TableNames: tables,
   504  		WithLock:   stmt.WithLock,
   505  		ForExport:  stmt.ForExport,
   506  	}
   507  }