vitess.io/vitess@v0.16.2/go/vt/vtgate/executor.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package vtgate
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/sha256"
    23  	"encoding/hex"
    24  	"encoding/json"
    25  	"errors"
    26  	"fmt"
    27  	"io"
    28  	"net/http"
    29  	"strings"
    30  	"sync"
    31  	"time"
    32  
    33  	"github.com/spf13/pflag"
    34  
    35  	"vitess.io/vitess/go/acl"
    36  	"vitess.io/vitess/go/cache"
    37  	"vitess.io/vitess/go/hack"
    38  	"vitess.io/vitess/go/mysql/collations"
    39  	"vitess.io/vitess/go/sqltypes"
    40  	"vitess.io/vitess/go/stats"
    41  	"vitess.io/vitess/go/sync2"
    42  	"vitess.io/vitess/go/trace"
    43  	"vitess.io/vitess/go/vt/callerid"
    44  	"vitess.io/vitess/go/vt/key"
    45  	"vitess.io/vitess/go/vt/log"
    46  	"vitess.io/vitess/go/vt/servenv"
    47  	"vitess.io/vitess/go/vt/sqlparser"
    48  	"vitess.io/vitess/go/vt/srvtopo"
    49  	"vitess.io/vitess/go/vt/sysvars"
    50  	"vitess.io/vitess/go/vt/topo/topoproto"
    51  	"vitess.io/vitess/go/vt/vterrors"
    52  	"vitess.io/vitess/go/vt/vtgate/engine"
    53  	"vitess.io/vitess/go/vt/vtgate/evalengine"
    54  	"vitess.io/vitess/go/vt/vtgate/logstats"
    55  	"vitess.io/vitess/go/vt/vtgate/planbuilder"
    56  	"vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext"
    57  	"vitess.io/vitess/go/vt/vtgate/vindexes"
    58  	"vitess.io/vitess/go/vt/vtgate/vschemaacl"
    59  
    60  	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
    61  	querypb "vitess.io/vitess/go/vt/proto/query"
    62  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    63  	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
    64  	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
    65  )
    66  
    67  var (
    68  	errNoKeyspace     = vterrors.VT09005()
    69  	defaultTabletType = topodatapb.TabletType_PRIMARY
    70  
    71  	// TODO: @rafael - These two counters should be deprecated in favor of the ByTable ones in v17+. They are kept for now for backwards compatibility.
    72  	queriesProcessed = stats.NewCountersWithSingleLabel("QueriesProcessed", "Queries processed at vtgate by plan type", "Plan")
    73  	queriesRouted    = stats.NewCountersWithSingleLabel("QueriesRouted", "Queries routed from vtgate to vttablet by plan type", "Plan")
    74  
    75  	queriesProcessedByTable = stats.NewCountersWithMultiLabels("QueriesProcessedByTable", "Queries processed at vtgate by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"})
    76  	queriesRoutedByTable    = stats.NewCountersWithMultiLabels("QueriesRoutedByTable", "Queries routed from vtgate to vttablet by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"})
    77  )
    78  
    79  const (
    80  	bindVarPrefix = "__vt"
    81  )
    82  
    83  func init() {
    84  	registerTabletTypeFlag := func(fs *pflag.FlagSet) {
    85  		fs.Var((*topoproto.TabletTypeFlag)(&defaultTabletType), "default_tablet_type", "The default tablet type to set for queries, when one is not explicitly selected.")
    86  	}
    87  
    88  	servenv.OnParseFor("vtgate", registerTabletTypeFlag)
    89  	servenv.OnParseFor("vtgateclienttest", registerTabletTypeFlag)
    90  	servenv.OnParseFor("vtcombo", registerTabletTypeFlag)
    91  	servenv.OnParseFor("vtexplain", registerTabletTypeFlag)
    92  }
    93  
    94  // Executor is the engine that executes queries by utilizing
    95  // the abilities of the underlying vttablets.
    96  type Executor struct {
    97  	serv        srvtopo.Server
    98  	cell        string
    99  	resolver    *Resolver
   100  	scatterConn *ScatterConn
   101  	txConn      *TxConn
   102  	pv          plancontext.PlannerVersion
   103  
   104  	mu           sync.Mutex
   105  	vschema      *vindexes.VSchema
   106  	streamSize   int
   107  	plans        cache.Cache
   108  	vschemaStats *VSchemaStats
   109  
   110  	normalize       bool
   111  	warnShardedOnly bool
   112  
   113  	vm            *VSchemaManager
   114  	schemaTracker SchemaInfo
   115  
   116  	// allowScatter will fail planning if set to false and a plan contains any scatter queries
   117  	allowScatter bool
   118  }
   119  
   120  var executorOnce sync.Once
   121  
   122  const pathQueryPlans = "/debug/query_plans"
   123  const pathScatterStats = "/debug/scatter_stats"
   124  const pathVSchema = "/debug/vschema"
   125  
   126  // NewExecutor creates a new Executor.
   127  func NewExecutor(
   128  	ctx context.Context,
   129  	serv srvtopo.Server,
   130  	cell string,
   131  	resolver *Resolver,
   132  	normalize, warnOnShardedOnly bool,
   133  	streamSize int,
   134  	cacheCfg *cache.Config,
   135  	schemaTracker SchemaInfo,
   136  	noScatter bool,
   137  	pv plancontext.PlannerVersion,
   138  ) *Executor {
   139  	e := &Executor{
   140  		serv:            serv,
   141  		cell:            cell,
   142  		resolver:        resolver,
   143  		scatterConn:     resolver.scatterConn,
   144  		txConn:          resolver.scatterConn.txConn,
   145  		plans:           cache.NewDefaultCacheImpl(cacheCfg),
   146  		normalize:       normalize,
   147  		warnShardedOnly: warnOnShardedOnly,
   148  		streamSize:      streamSize,
   149  		schemaTracker:   schemaTracker,
   150  		allowScatter:    !noScatter,
   151  		pv:              pv,
   152  	}
   153  
   154  	vschemaacl.Init()
   155  	// we subscribe to update from the VSchemaManager
   156  	e.vm = &VSchemaManager{
   157  		subscriber: e.SaveVSchema,
   158  		serv:       serv,
   159  		cell:       cell,
   160  		schema:     e.schemaTracker,
   161  	}
   162  	serv.WatchSrvVSchema(ctx, cell, e.vm.VSchemaUpdate)
   163  
   164  	executorOnce.Do(func() {
   165  		stats.NewGaugeFunc("QueryPlanCacheLength", "Query plan cache length", func() int64 {
   166  			return int64(e.plans.Len())
   167  		})
   168  		stats.NewGaugeFunc("QueryPlanCacheSize", "Query plan cache size", func() int64 {
   169  			return e.plans.UsedCapacity()
   170  		})
   171  		stats.NewGaugeFunc("QueryPlanCacheCapacity", "Query plan cache capacity", func() int64 {
   172  			return e.plans.MaxCapacity()
   173  		})
   174  		stats.NewCounterFunc("QueryPlanCacheEvictions", "Query plan cache evictions", func() int64 {
   175  			return e.plans.Evictions()
   176  		})
   177  		stats.NewCounterFunc("QueryPlanCacheHits", "Query plan cache hits", func() int64 {
   178  			return e.plans.Hits()
   179  		})
   180  		stats.NewCounterFunc("QueryPlanCacheMisses", "Query plan cache misses", func() int64 {
   181  			return e.plans.Misses()
   182  		})
   183  		http.Handle(pathQueryPlans, e)
   184  		http.Handle(pathScatterStats, e)
   185  		http.Handle(pathVSchema, e)
   186  	})
   187  	return e
   188  }
   189  
   190  // Execute executes a non-streaming query.
   191  func (e *Executor) Execute(ctx context.Context, method string, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable) (result *sqltypes.Result, err error) {
   192  	span, ctx := trace.NewSpan(ctx, "executor.Execute")
   193  	span.Annotate("method", method)
   194  	trace.AnnotateSQL(span, sqlparser.Preview(sql))
   195  	defer span.Finish()
   196  
   197  	logStats := logstats.NewLogStats(ctx, method, sql, safeSession.GetSessionUUID(), bindVars)
   198  	stmtType, result, err := e.execute(ctx, safeSession, sql, bindVars, logStats)
   199  	logStats.Error = err
   200  	if result == nil {
   201  		saveSessionStats(safeSession, stmtType, 0, 0, 0, err)
   202  	} else {
   203  		saveSessionStats(safeSession, stmtType, result.RowsAffected, result.InsertID, len(result.Rows), err)
   204  	}
   205  	if result != nil && len(result.Rows) > warnMemoryRows {
   206  		warnings.Add("ResultsExceeded", 1)
   207  		piiSafeSQL, err := sqlparser.RedactSQLQuery(sql)
   208  		if err != nil {
   209  			piiSafeSQL = logStats.StmtType
   210  		}
   211  		log.Warningf("%q exceeds warning threshold of max memory rows: %v", piiSafeSQL, warnMemoryRows)
   212  	}
   213  
   214  	logStats.SaveEndTime()
   215  	QueryLogger.Send(logStats)
   216  	return result, err
   217  }
   218  
   219  type streaminResultReceiver struct {
   220  	mu           sync.Mutex
   221  	stmtType     sqlparser.StatementType
   222  	rowsAffected uint64
   223  	rowsReturned int
   224  	insertID     uint64
   225  	callback     func(*sqltypes.Result) error
   226  }
   227  
   228  func (s *streaminResultReceiver) storeResultStats(typ sqlparser.StatementType, qr *sqltypes.Result) error {
   229  	s.mu.Lock()
   230  	defer s.mu.Unlock()
   231  	s.rowsAffected += qr.RowsAffected
   232  	s.rowsReturned += len(qr.Rows)
   233  	if qr.InsertID != 0 {
   234  		s.insertID = qr.InsertID
   235  	}
   236  	s.stmtType = typ
   237  	return s.callback(qr)
   238  }
   239  
   240  // StreamExecute executes a streaming query.
   241  func (e *Executor) StreamExecute(
   242  	ctx context.Context,
   243  	method string,
   244  	safeSession *SafeSession,
   245  	sql string,
   246  	bindVars map[string]*querypb.BindVariable,
   247  	callback func(*sqltypes.Result) error,
   248  ) error {
   249  	span, ctx := trace.NewSpan(ctx, "executor.StreamExecute")
   250  	span.Annotate("method", method)
   251  	trace.AnnotateSQL(span, sqlparser.Preview(sql))
   252  	defer span.Finish()
   253  
   254  	logStats := logstats.NewLogStats(ctx, method, sql, safeSession.GetSessionUUID(), bindVars)
   255  	srr := &streaminResultReceiver{callback: callback}
   256  	var err error
   257  
   258  	resultHandler := func(ctx context.Context, plan *engine.Plan, vc *vcursorImpl, bindVars map[string]*querypb.BindVariable, execStart time.Time) error {
   259  		var seenResults sync2.AtomicBool
   260  		var resultMu sync.Mutex
   261  		result := &sqltypes.Result{}
   262  		if canReturnRows(plan.Type) {
   263  			srr.callback = func(qr *sqltypes.Result) error {
   264  				resultMu.Lock()
   265  				defer resultMu.Unlock()
   266  				// If the row has field info, send it separately.
   267  				// TODO(sougou): this behavior is for handling tests because
   268  				// the framework currently sends all results as one packet.
   269  				byteCount := 0
   270  				if len(qr.Fields) > 0 {
   271  					if err := callback(qr.Metadata()); err != nil {
   272  						return err
   273  					}
   274  					seenResults.Set(true)
   275  				}
   276  
   277  				for _, row := range qr.Rows {
   278  					result.Rows = append(result.Rows, row)
   279  
   280  					for _, col := range row {
   281  						byteCount += col.Len()
   282  					}
   283  
   284  					if byteCount >= e.streamSize {
   285  						err := callback(result)
   286  						seenResults.Set(true)
   287  						result = &sqltypes.Result{}
   288  						byteCount = 0
   289  						if err != nil {
   290  							return err
   291  						}
   292  					}
   293  				}
   294  				return nil
   295  			}
   296  		}
   297  
   298  		// 4: Execute!
   299  		err := vc.StreamExecutePrimitive(ctx, plan.Instructions, bindVars, true, func(qr *sqltypes.Result) error {
   300  			return srr.storeResultStats(plan.Type, qr)
   301  		})
   302  
   303  		// Check if there was partial DML execution. If so, rollback the effect of the partially executed query.
   304  		if err != nil {
   305  			if !canReturnRows(plan.Type) {
   306  				return e.rollbackExecIfNeeded(ctx, safeSession, bindVars, logStats, err)
   307  			}
   308  			return err
   309  		}
   310  
   311  		if !canReturnRows(plan.Type) {
   312  			return nil
   313  		}
   314  
   315  		// Send left-over rows if there is no error on execution.
   316  		if len(result.Rows) > 0 || !seenResults.Get() {
   317  			if err := callback(result); err != nil {
   318  				return err
   319  			}
   320  		}
   321  
   322  		// 5: Log and add statistics
   323  		logStats.TablesUsed = plan.TablesUsed
   324  		logStats.TabletType = vc.TabletType().String()
   325  		logStats.ExecuteTime = time.Since(execStart)
   326  		logStats.ActiveKeyspace = vc.keyspace
   327  
   328  		e.updateQueryCounts(plan.Instructions.RouteType(), plan.Instructions.GetKeyspaceName(), plan.Instructions.GetTableName(), int64(logStats.ShardQueries))
   329  
   330  		return err
   331  	}
   332  
   333  	err = e.newExecute(ctx, safeSession, sql, bindVars, logStats, resultHandler, srr.storeResultStats)
   334  
   335  	logStats.Error = err
   336  	saveSessionStats(safeSession, srr.stmtType, srr.rowsAffected, srr.insertID, srr.rowsReturned, err)
   337  	if srr.rowsReturned > warnMemoryRows {
   338  		warnings.Add("ResultsExceeded", 1)
   339  		piiSafeSQL, err := sqlparser.RedactSQLQuery(sql)
   340  		if err != nil {
   341  			piiSafeSQL = logStats.StmtType
   342  		}
   343  		log.Warningf("%q exceeds warning threshold of max memory rows: %v", piiSafeSQL, warnMemoryRows)
   344  	}
   345  
   346  	logStats.SaveEndTime()
   347  	QueryLogger.Send(logStats)
   348  	return err
   349  
   350  }
   351  
   352  func canReturnRows(stmtType sqlparser.StatementType) bool {
   353  	switch stmtType {
   354  	case sqlparser.StmtSelect, sqlparser.StmtShow, sqlparser.StmtExplain, sqlparser.StmtCallProc:
   355  		return true
   356  	default:
   357  		return false
   358  	}
   359  }
   360  
   361  func saveSessionStats(safeSession *SafeSession, stmtType sqlparser.StatementType, rowsAffected, insertID uint64, rowsReturned int, err error) {
   362  	safeSession.RowCount = -1
   363  	if err != nil {
   364  		return
   365  	}
   366  	if !safeSession.foundRowsHandled {
   367  		safeSession.FoundRows = uint64(rowsReturned)
   368  	}
   369  	if insertID > 0 {
   370  		safeSession.LastInsertId = insertID
   371  	}
   372  	switch stmtType {
   373  	case sqlparser.StmtInsert, sqlparser.StmtReplace, sqlparser.StmtUpdate, sqlparser.StmtDelete:
   374  		safeSession.RowCount = int64(rowsAffected)
   375  	case sqlparser.StmtDDL, sqlparser.StmtSet, sqlparser.StmtBegin, sqlparser.StmtCommit, sqlparser.StmtRollback, sqlparser.StmtFlush:
   376  		safeSession.RowCount = 0
   377  	}
   378  }
   379  
   380  func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) (sqlparser.StatementType, *sqltypes.Result, error) {
   381  	var err error
   382  	var qr *sqltypes.Result
   383  	var stmtType sqlparser.StatementType
   384  	err = e.newExecute(ctx, safeSession, sql, bindVars, logStats, func(ctx context.Context, plan *engine.Plan, vc *vcursorImpl, bindVars map[string]*querypb.BindVariable, time time.Time) error {
   385  		stmtType = plan.Type
   386  		qr, err = e.executePlan(ctx, safeSession, plan, vc, bindVars, logStats, time)
   387  		return err
   388  	}, func(typ sqlparser.StatementType, result *sqltypes.Result) error {
   389  		stmtType = typ
   390  		qr = result
   391  		return nil
   392  	})
   393  
   394  	return stmtType, qr, err
   395  }
   396  
   397  // addNeededBindVars adds bind vars that are needed by the plan
   398  func (e *Executor) addNeededBindVars(bindVarNeeds *sqlparser.BindVarNeeds, bindVars map[string]*querypb.BindVariable, session *SafeSession) error {
   399  	for _, funcName := range bindVarNeeds.NeedFunctionResult {
   400  		switch funcName {
   401  		case sqlparser.DBVarName:
   402  			bindVars[sqlparser.DBVarName] = sqltypes.StringBindVariable(session.TargetString)
   403  		case sqlparser.LastInsertIDName:
   404  			bindVars[sqlparser.LastInsertIDName] = sqltypes.Uint64BindVariable(session.GetLastInsertId())
   405  		case sqlparser.FoundRowsName:
   406  			bindVars[sqlparser.FoundRowsName] = sqltypes.Int64BindVariable(int64(session.FoundRows))
   407  		case sqlparser.RowCountName:
   408  			bindVars[sqlparser.RowCountName] = sqltypes.Int64BindVariable(session.RowCount)
   409  		}
   410  	}
   411  
   412  	for _, sysVar := range bindVarNeeds.NeedSystemVariable {
   413  		key := bindVarPrefix + sysVar
   414  		switch sysVar {
   415  		case sysvars.Autocommit.Name:
   416  			bindVars[key] = sqltypes.BoolBindVariable(session.Autocommit)
   417  		case sysvars.QueryTimeout.Name:
   418  			bindVars[key] = sqltypes.Int64BindVariable(session.GetQueryTimeout())
   419  		case sysvars.ClientFoundRows.Name:
   420  			var v bool
   421  			ifOptionsExist(session, func(options *querypb.ExecuteOptions) {
   422  				v = options.ClientFoundRows
   423  			})
   424  			bindVars[key] = sqltypes.BoolBindVariable(v)
   425  		case sysvars.SkipQueryPlanCache.Name:
   426  			var v bool
   427  			ifOptionsExist(session, func(options *querypb.ExecuteOptions) {
   428  				v = options.ClientFoundRows
   429  			})
   430  			bindVars[key] = sqltypes.BoolBindVariable(v)
   431  		case sysvars.SQLSelectLimit.Name:
   432  			var v int64
   433  			ifOptionsExist(session, func(options *querypb.ExecuteOptions) {
   434  				v = options.SqlSelectLimit
   435  			})
   436  			bindVars[key] = sqltypes.Int64BindVariable(v)
   437  		case sysvars.TransactionMode.Name:
   438  			bindVars[key] = sqltypes.StringBindVariable(session.TransactionMode.String())
   439  		case sysvars.Workload.Name:
   440  			var v string
   441  			ifOptionsExist(session, func(options *querypb.ExecuteOptions) {
   442  				v = options.GetWorkload().String()
   443  			})
   444  			bindVars[key] = sqltypes.StringBindVariable(v)
   445  		case sysvars.DDLStrategy.Name:
   446  			bindVars[key] = sqltypes.StringBindVariable(session.DDLStrategy)
   447  		case sysvars.SessionUUID.Name:
   448  			bindVars[key] = sqltypes.StringBindVariable(session.SessionUUID)
   449  		case sysvars.SessionEnableSystemSettings.Name:
   450  			bindVars[key] = sqltypes.BoolBindVariable(session.EnableSystemSettings)
   451  		case sysvars.ReadAfterWriteGTID.Name:
   452  			var v string
   453  			ifReadAfterWriteExist(session, func(raw *vtgatepb.ReadAfterWrite) {
   454  				v = raw.ReadAfterWriteGtid
   455  			})
   456  			bindVars[key] = sqltypes.StringBindVariable(v)
   457  		case sysvars.ReadAfterWriteTimeOut.Name:
   458  			var v float64
   459  			ifReadAfterWriteExist(session, func(raw *vtgatepb.ReadAfterWrite) {
   460  				v = raw.ReadAfterWriteTimeout
   461  			})
   462  			bindVars[key] = sqltypes.Float64BindVariable(v)
   463  		case sysvars.SessionTrackGTIDs.Name:
   464  			v := "off"
   465  			ifReadAfterWriteExist(session, func(raw *vtgatepb.ReadAfterWrite) {
   466  				if raw.SessionTrackGtids {
   467  					v = "own_gtid"
   468  				}
   469  			})
   470  			bindVars[key] = sqltypes.StringBindVariable(v)
   471  		case sysvars.Version.Name:
   472  			bindVars[key] = sqltypes.StringBindVariable(servenv.AppVersion.MySQLVersion())
   473  		case sysvars.VersionComment.Name:
   474  			bindVars[key] = sqltypes.StringBindVariable(servenv.AppVersion.String())
   475  		case sysvars.Socket.Name:
   476  			bindVars[key] = sqltypes.StringBindVariable(mysqlSocketPath())
   477  		default:
   478  			if value, hasSysVar := session.SystemVariables[sysVar]; hasSysVar {
   479  				expr, err := sqlparser.ParseExpr(value)
   480  				if err != nil {
   481  					return err
   482  				}
   483  
   484  				evalExpr, err := evalengine.Translate(expr, nil)
   485  				if err != nil {
   486  					return err
   487  				}
   488  				evaluated, err := evalengine.EmptyExpressionEnv().Evaluate(evalExpr)
   489  				if err != nil {
   490  					return err
   491  				}
   492  				bindVars[key] = sqltypes.ValueBindVariable(evaluated.Value())
   493  			}
   494  		}
   495  	}
   496  
   497  	udvMap := session.UserDefinedVariables
   498  	if udvMap == nil {
   499  		udvMap = map[string]*querypb.BindVariable{}
   500  	}
   501  	for _, udv := range bindVarNeeds.NeedUserDefinedVariables {
   502  		val := udvMap[udv]
   503  		if val == nil {
   504  			val = sqltypes.NullBindVariable
   505  		}
   506  		bindVars[sqlparser.UserDefinedVariableName+udv] = val
   507  	}
   508  
   509  	return nil
   510  }
   511  
   512  func ifOptionsExist(session *SafeSession, f func(*querypb.ExecuteOptions)) {
   513  	options := session.GetOptions()
   514  	if options != nil {
   515  		f(options)
   516  	}
   517  }
   518  
   519  func ifReadAfterWriteExist(session *SafeSession, f func(*vtgatepb.ReadAfterWrite)) {
   520  	raw := session.ReadAfterWrite
   521  	if raw != nil {
   522  		f(raw)
   523  	}
   524  }
   525  
   526  func (e *Executor) handleBegin(ctx context.Context, safeSession *SafeSession, logStats *logstats.LogStats, stmt sqlparser.Statement) (*sqltypes.Result, error) {
   527  	execStart := time.Now()
   528  	logStats.PlanTime = execStart.Sub(logStats.StartTime)
   529  
   530  	begin := stmt.(*sqlparser.Begin)
   531  	err := e.txConn.Begin(ctx, safeSession, begin.TxAccessModes)
   532  	logStats.ExecuteTime = time.Since(execStart)
   533  
   534  	e.updateQueryCounts("Begin", "", "", 0)
   535  
   536  	return &sqltypes.Result{}, err
   537  }
   538  
   539  func (e *Executor) handleCommit(ctx context.Context, safeSession *SafeSession, logStats *logstats.LogStats) (*sqltypes.Result, error) {
   540  	execStart := time.Now()
   541  	logStats.PlanTime = execStart.Sub(logStats.StartTime)
   542  	logStats.ShardQueries = uint64(len(safeSession.ShardSessions))
   543  	e.updateQueryCounts("Commit", "", "", int64(logStats.ShardQueries))
   544  
   545  	err := e.txConn.Commit(ctx, safeSession)
   546  	logStats.CommitTime = time.Since(execStart)
   547  	return &sqltypes.Result{}, err
   548  }
   549  
   550  // Commit commits the existing transactions
   551  func (e *Executor) Commit(ctx context.Context, safeSession *SafeSession) error {
   552  	return e.txConn.Commit(ctx, safeSession)
   553  }
   554  
   555  func (e *Executor) handleRollback(ctx context.Context, safeSession *SafeSession, logStats *logstats.LogStats) (*sqltypes.Result, error) {
   556  	execStart := time.Now()
   557  	logStats.PlanTime = execStart.Sub(logStats.StartTime)
   558  	logStats.ShardQueries = uint64(len(safeSession.ShardSessions))
   559  	e.updateQueryCounts("Rollback", "", "", int64(logStats.ShardQueries))
   560  	err := e.txConn.Rollback(ctx, safeSession)
   561  	logStats.CommitTime = time.Since(execStart)
   562  	return &sqltypes.Result{}, err
   563  }
   564  
   565  func (e *Executor) handleSavepoint(ctx context.Context, safeSession *SafeSession, sql string, planType string, logStats *logstats.LogStats, nonTxResponse func(query string) (*sqltypes.Result, error), ignoreMaxMemoryRows bool) (*sqltypes.Result, error) {
   566  	execStart := time.Now()
   567  	logStats.PlanTime = execStart.Sub(logStats.StartTime)
   568  	logStats.ShardQueries = uint64(len(safeSession.ShardSessions))
   569  	e.updateQueryCounts(planType, "", "", int64(logStats.ShardQueries))
   570  	defer func() {
   571  		logStats.ExecuteTime = time.Since(execStart)
   572  	}()
   573  
   574  	// If no transaction exists on any of the shard sessions,
   575  	// then savepoint does not need to be executed, it will be only stored in the session
   576  	// and later will be executed when a transaction is started.
   577  	if !safeSession.isTxOpen() {
   578  		if safeSession.InTransaction() {
   579  			// Storing, as this needs to be executed just after starting transaction on the shard.
   580  			safeSession.StoreSavepoint(sql)
   581  			return &sqltypes.Result{}, nil
   582  		}
   583  		return nonTxResponse(sql)
   584  	}
   585  	orig := safeSession.commitOrder
   586  	qr, err := e.executeSPInAllSessions(ctx, safeSession, sql, ignoreMaxMemoryRows)
   587  	safeSession.SetCommitOrder(orig)
   588  	if err != nil {
   589  		return nil, err
   590  	}
   591  	safeSession.StoreSavepoint(sql)
   592  	return qr, nil
   593  }
   594  
   595  // executeSPInAllSessions function executes the savepoint query in all open shard sessions (pre, normal and post)
   596  // which has non-zero transaction id (i.e. an open transaction on the shard connection).
   597  func (e *Executor) executeSPInAllSessions(ctx context.Context, safeSession *SafeSession, sql string, ignoreMaxMemoryRows bool) (*sqltypes.Result, error) {
   598  	var qr *sqltypes.Result
   599  	var errs []error
   600  	for _, co := range []vtgatepb.CommitOrder{vtgatepb.CommitOrder_PRE, vtgatepb.CommitOrder_NORMAL, vtgatepb.CommitOrder_POST} {
   601  		safeSession.SetCommitOrder(co)
   602  
   603  		var rss []*srvtopo.ResolvedShard
   604  		var queries []*querypb.BoundQuery
   605  		for _, shardSession := range safeSession.getSessions() {
   606  			// This will avoid executing savepoint on reserved connections
   607  			// which has no open transaction.
   608  			if shardSession.TransactionId == 0 {
   609  				continue
   610  			}
   611  			rss = append(rss, &srvtopo.ResolvedShard{
   612  				Target:  shardSession.Target,
   613  				Gateway: e.resolver.resolver.GetGateway(),
   614  			})
   615  			queries = append(queries, &querypb.BoundQuery{Sql: sql})
   616  		}
   617  		qr, errs = e.ExecuteMultiShard(ctx, nil, rss, queries, safeSession, false /*autocommit*/, ignoreMaxMemoryRows)
   618  		err := vterrors.Aggregate(errs)
   619  		if err != nil {
   620  			return nil, err
   621  		}
   622  	}
   623  	return qr, nil
   624  }
   625  
   626  // CloseSession releases the current connection, which rollbacks open transactions and closes reserved connections.
   627  // It is called then the MySQL servers closes the connection to its client.
   628  func (e *Executor) CloseSession(ctx context.Context, safeSession *SafeSession) error {
   629  	return e.txConn.ReleaseAll(ctx, safeSession)
   630  }
   631  
   632  func (e *Executor) setVitessMetadata(ctx context.Context, name, value string) error {
   633  	// TODO(kalfonso): move to its own acl check and consolidate into an acl component that can handle multiple operations (vschema, metadata)
   634  	user := callerid.ImmediateCallerIDFromContext(ctx)
   635  	allowed := vschemaacl.Authorized(user)
   636  	if !allowed {
   637  		return vterrors.NewErrorf(vtrpcpb.Code_PERMISSION_DENIED, vterrors.AccessDeniedError, "User '%s' not authorized to perform vitess metadata operations", user.GetUsername())
   638  	}
   639  
   640  	ts, err := e.serv.GetTopoServer()
   641  	if err != nil {
   642  		return err
   643  	}
   644  
   645  	if value == "" {
   646  		return ts.DeleteMetadata(ctx, name)
   647  	}
   648  	return ts.UpsertMetadata(ctx, name, value)
   649  }
   650  
   651  func (e *Executor) showVitessMetadata(ctx context.Context, filter *sqlparser.ShowFilter) (*sqltypes.Result, error) {
   652  	ts, err := e.serv.GetTopoServer()
   653  	if err != nil {
   654  		return nil, err
   655  	}
   656  
   657  	var metadata map[string]string
   658  	if filter == nil {
   659  		metadata, err = ts.GetMetadata(ctx, "")
   660  		if err != nil {
   661  			return nil, err
   662  		}
   663  	} else {
   664  		metadata, err = ts.GetMetadata(ctx, filter.Like)
   665  		if err != nil {
   666  			return nil, err
   667  		}
   668  	}
   669  
   670  	rows := make([][]sqltypes.Value, 0, len(metadata))
   671  	for k, v := range metadata {
   672  		row := buildVarCharRow(k, v)
   673  		rows = append(rows, row)
   674  	}
   675  
   676  	return &sqltypes.Result{
   677  		Fields: buildVarCharFields("Key", "Value"),
   678  		Rows:   rows,
   679  	}, nil
   680  }
   681  
   682  type tabletFilter func(tablet *topodatapb.Tablet, servingState string, primaryTermStartTime int64) bool
   683  
   684  func (e *Executor) showShards(ctx context.Context, filter *sqlparser.ShowFilter, destTabletType topodatapb.TabletType) (*sqltypes.Result, error) {
   685  	showVitessShardsFilters := func(filter *sqlparser.ShowFilter) ([]func(string) bool, []func(string, *topodatapb.ShardReference) bool) {
   686  		keyspaceFilters := []func(string) bool{}
   687  		shardFilters := []func(string, *topodatapb.ShardReference) bool{}
   688  
   689  		if filter == nil {
   690  			return keyspaceFilters, shardFilters
   691  		}
   692  
   693  		if filter.Like != "" {
   694  			shardLikeRexep := sqlparser.LikeToRegexp(filter.Like)
   695  
   696  			if strings.Contains(filter.Like, "/") {
   697  				keyspaceLikeRexep := sqlparser.LikeToRegexp(strings.Split(filter.Like, "/")[0])
   698  				keyspaceFilters = append(keyspaceFilters, func(ks string) bool {
   699  					return keyspaceLikeRexep.MatchString(ks)
   700  				})
   701  			}
   702  			shardFilters = append(shardFilters, func(ks string, shard *topodatapb.ShardReference) bool {
   703  				return shardLikeRexep.MatchString(topoproto.KeyspaceShardString(ks, shard.Name))
   704  			})
   705  
   706  			return keyspaceFilters, shardFilters
   707  		}
   708  
   709  		if filter.Filter != nil {
   710  			// TODO build a query planner I guess? lol that should be fun
   711  			log.Infof("SHOW VITESS_SHARDS where clause %+v. Ignoring this (for now).", filter.Filter)
   712  		}
   713  
   714  		return keyspaceFilters, shardFilters
   715  	}
   716  
   717  	keyspaceFilters, shardFilters := showVitessShardsFilters(filter)
   718  
   719  	keyspaces, err := e.resolver.resolver.GetAllKeyspaces(ctx)
   720  	if err != nil {
   721  		return nil, err
   722  	}
   723  
   724  	var rows [][]sqltypes.Value
   725  	for _, keyspace := range keyspaces {
   726  		skipKeyspace := false
   727  		for _, filter := range keyspaceFilters {
   728  			if !filter(keyspace) {
   729  				skipKeyspace = true
   730  				break
   731  			}
   732  		}
   733  
   734  		if skipKeyspace {
   735  			continue
   736  		}
   737  
   738  		_, _, shards, err := e.resolver.resolver.GetKeyspaceShards(ctx, keyspace, destTabletType)
   739  		if err != nil {
   740  			// There might be a misconfigured keyspace or no shards in the keyspace.
   741  			// Skip any errors and move on.
   742  			continue
   743  		}
   744  
   745  		for _, shard := range shards {
   746  			skipShard := false
   747  			for _, filter := range shardFilters {
   748  				if !filter(keyspace, shard) {
   749  					skipShard = true
   750  					break
   751  				}
   752  			}
   753  
   754  			if skipShard {
   755  				continue
   756  			}
   757  
   758  			rows = append(rows, buildVarCharRow(topoproto.KeyspaceShardString(keyspace, shard.Name)))
   759  		}
   760  	}
   761  
   762  	return &sqltypes.Result{
   763  		Fields: buildVarCharFields("Shards"),
   764  		Rows:   rows,
   765  	}, nil
   766  }
   767  
   768  func (e *Executor) showTablets(filter *sqlparser.ShowFilter) (*sqltypes.Result, error) {
   769  	getTabletFilters := func(filter *sqlparser.ShowFilter) []tabletFilter {
   770  		var filters []tabletFilter
   771  
   772  		if filter == nil {
   773  			return filters
   774  		}
   775  
   776  		if filter.Like != "" {
   777  			tabletRegexp := sqlparser.LikeToRegexp(filter.Like)
   778  
   779  			f := func(tablet *topodatapb.Tablet, servingState string, primaryTermStartTime int64) bool {
   780  				return tabletRegexp.MatchString(tablet.Hostname)
   781  			}
   782  
   783  			filters = append(filters, f)
   784  			return filters
   785  		}
   786  
   787  		if filter.Filter != nil {
   788  			log.Infof("SHOW VITESS_TABLETS where clause: %+v. Ignoring this (for now).", filter.Filter)
   789  		}
   790  
   791  		return filters
   792  	}
   793  
   794  	tabletFilters := getTabletFilters(filter)
   795  
   796  	rows := [][]sqltypes.Value{}
   797  	status := e.scatterConn.GetHealthCheckCacheStatus()
   798  	for _, s := range status {
   799  		for _, ts := range s.TabletsStats {
   800  			state := "SERVING"
   801  			if !ts.Serving {
   802  				state = "NOT_SERVING"
   803  			}
   804  			ptst := ts.PrimaryTermStartTime
   805  			ptstStr := ""
   806  			if ptst > 0 {
   807  				// this code depends on the fact that PrimaryTermStartTime is the seconds since epoch start
   808  				ptstStr = time.Unix(ptst, 0).UTC().Format(time.RFC3339)
   809  			}
   810  
   811  			skipTablet := false
   812  			for _, filter := range tabletFilters {
   813  				if !filter(ts.Tablet, state, ptst) {
   814  					skipTablet = true
   815  					break
   816  				}
   817  			}
   818  
   819  			if skipTablet {
   820  				continue
   821  			}
   822  
   823  			rows = append(rows, buildVarCharRow(
   824  				s.Cell,
   825  				s.Target.Keyspace,
   826  				s.Target.Shard,
   827  				ts.Target.TabletType.String(),
   828  				state,
   829  				topoproto.TabletAliasString(ts.Tablet.Alias),
   830  				ts.Tablet.Hostname,
   831  				ptstStr,
   832  			))
   833  		}
   834  	}
   835  	return &sqltypes.Result{
   836  		Fields: buildVarCharFields("Cell", "Keyspace", "Shard", "TabletType", "State", "Alias", "Hostname", "PrimaryTermStartTime"),
   837  		Rows:   rows,
   838  	}, nil
   839  }
   840  
   841  func (e *Executor) showVitessReplicationStatus(ctx context.Context, filter *sqlparser.ShowFilter) (*sqltypes.Result, error) {
   842  	ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
   843  	defer cancel()
   844  	rows := [][]sqltypes.Value{}
   845  
   846  	status := e.scatterConn.GetHealthCheckCacheStatus()
   847  
   848  	for _, s := range status {
   849  		for _, ts := range s.TabletsStats {
   850  			// We only want to show REPLICA and RDONLY tablets
   851  			if ts.Tablet.Type != topodatapb.TabletType_REPLICA && ts.Tablet.Type != topodatapb.TabletType_RDONLY {
   852  				continue
   853  			}
   854  
   855  			// Allow people to filter by Keyspace and Shard using a LIKE clause
   856  			if filter != nil {
   857  				ksFilterRegex := sqlparser.LikeToRegexp(filter.Like)
   858  				keyspaceShardStr := fmt.Sprintf("%s/%s", ts.Tablet.Keyspace, ts.Tablet.Shard)
   859  				if !ksFilterRegex.MatchString(keyspaceShardStr) {
   860  					continue
   861  				}
   862  			}
   863  
   864  			tabletHostPort := ts.GetTabletHostPort()
   865  			throttlerStatus, err := getTabletThrottlerStatus(tabletHostPort)
   866  			if err != nil {
   867  				log.Warningf("Could not get throttler status from %s: %v", tabletHostPort, err)
   868  			}
   869  
   870  			replSourceHost := ""
   871  			replSourcePort := int64(0)
   872  			replIOThreadHealth := ""
   873  			replSQLThreadHealth := ""
   874  			replLastError := ""
   875  			replLag := int64(-1)
   876  			sql := "show slave status"
   877  			results, err := e.txConn.tabletGateway.Execute(ctx, ts.Target, sql, nil, 0, 0, nil)
   878  			if err != nil || results == nil {
   879  				log.Warningf("Could not get replication status from %s: %v", tabletHostPort, err)
   880  			} else if row := results.Named().Row(); row != nil {
   881  				replSourceHost = row["Master_Host"].ToString()
   882  				replSourcePort, _ = row["Master_Port"].ToInt64()
   883  				replIOThreadHealth = row["Slave_IO_Running"].ToString()
   884  				replSQLThreadHealth = row["Slave_SQL_Running"].ToString()
   885  				replLastError = row["Last_Error"].ToString()
   886  				if ts.Stats != nil {
   887  					replLag = int64(ts.Stats.ReplicationLagSeconds)
   888  				}
   889  			}
   890  			replicationHealth := fmt.Sprintf("{\"EventStreamRunning\":\"%s\",\"EventApplierRunning\":\"%s\",\"LastError\":\"%s\"}", replIOThreadHealth, replSQLThreadHealth, replLastError)
   891  
   892  			rows = append(rows, buildVarCharRow(
   893  				s.Target.Keyspace,
   894  				s.Target.Shard,
   895  				ts.Target.TabletType.String(),
   896  				topoproto.TabletAliasString(ts.Tablet.Alias),
   897  				ts.Tablet.Hostname,
   898  				fmt.Sprintf("%s:%d", replSourceHost, replSourcePort),
   899  				replicationHealth,
   900  				fmt.Sprintf("%d", replLag),
   901  				throttlerStatus,
   902  			))
   903  		}
   904  	}
   905  	return &sqltypes.Result{
   906  		Fields: buildVarCharFields("Keyspace", "Shard", "TabletType", "Alias", "Hostname", "ReplicationSource", "ReplicationHealth", "ReplicationLag", "ThrottlerStatus"),
   907  		Rows:   rows,
   908  	}, nil
   909  }
   910  
   911  // MessageStream is part of the vtgate service API. This is a V2 level API that's sent
   912  // to the Resolver.
   913  func (e *Executor) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error {
   914  	err := e.resolver.MessageStream(
   915  		ctx,
   916  		keyspace,
   917  		shard,
   918  		keyRange,
   919  		name,
   920  		callback,
   921  	)
   922  	return formatError(err)
   923  }
   924  
   925  // VSchema returns the VSchema.
   926  func (e *Executor) VSchema() *vindexes.VSchema {
   927  	e.mu.Lock()
   928  	defer e.mu.Unlock()
   929  	return e.vschema
   930  }
   931  
   932  // SaveVSchema updates the vschema and stats
   933  func (e *Executor) SaveVSchema(vschema *vindexes.VSchema, stats *VSchemaStats) {
   934  	e.mu.Lock()
   935  	defer e.mu.Unlock()
   936  	if vschema != nil {
   937  		e.vschema = vschema
   938  	}
   939  	e.vschemaStats = stats
   940  	e.plans.Clear()
   941  
   942  	if vschemaCounters != nil {
   943  		vschemaCounters.Add("Reload", 1)
   944  	}
   945  
   946  }
   947  
   948  // ParseDestinationTarget parses destination target string and sets default keyspace if possible.
   949  func (e *Executor) ParseDestinationTarget(targetString string) (string, topodatapb.TabletType, key.Destination, error) {
   950  	destKeyspace, destTabletType, dest, err := topoproto.ParseDestination(targetString, defaultTabletType)
   951  	// Set default keyspace
   952  	if destKeyspace == "" && len(e.VSchema().Keyspaces) == 1 {
   953  		for k := range e.VSchema().Keyspaces {
   954  			destKeyspace = k
   955  		}
   956  	}
   957  	return destKeyspace, destTabletType, dest, err
   958  }
   959  
   960  type iQueryOption interface {
   961  	cachePlan() bool
   962  	getSelectLimit() int
   963  }
   964  
   965  // getPlan computes the plan for the given query. If one is in
   966  // the cache, it reuses it.
   967  func (e *Executor) getPlan(ctx context.Context, vcursor *vcursorImpl, sql string, comments sqlparser.MarginComments, bindVars map[string]*querypb.BindVariable, qo iQueryOption, logStats *logstats.LogStats) (*engine.Plan, sqlparser.Statement, error) {
   968  	if e.VSchema() == nil {
   969  		return nil, nil, errors.New("vschema not initialized")
   970  	}
   971  
   972  	stmt, reserved, err := sqlparser.Parse2(sql)
   973  	if err != nil {
   974  		return nil, nil, err
   975  	}
   976  	query := sql
   977  	statement := stmt
   978  	reservedVars := sqlparser.NewReservedVars("vtg", reserved)
   979  	bindVarNeeds := &sqlparser.BindVarNeeds{}
   980  	if !sqlparser.IgnoreMaxPayloadSizeDirective(statement) && !isValidPayloadSize(query) {
   981  		return nil, nil, vterrors.NewErrorf(vtrpcpb.Code_RESOURCE_EXHAUSTED, vterrors.NetPacketTooLarge, "query payload size above threshold")
   982  	}
   983  	ignoreMaxMemoryRows := sqlparser.IgnoreMaxMaxMemoryRowsDirective(stmt)
   984  	vcursor.SetIgnoreMaxMemoryRows(ignoreMaxMemoryRows)
   985  	consolidator := sqlparser.Consolidator(stmt)
   986  	vcursor.SetConsolidator(consolidator)
   987  
   988  	setVarComment, err := prepareSetVarComment(vcursor, stmt)
   989  	if err != nil {
   990  		return nil, nil, err
   991  	}
   992  	// Normalize if possible and retry.
   993  	if e.canNormalizeStatement(stmt, qo, setVarComment) {
   994  		parameterize := e.normalize // the public flag is called normalize
   995  		result, err := sqlparser.PrepareAST(
   996  			stmt,
   997  			reservedVars,
   998  			bindVars,
   999  			parameterize,
  1000  			vcursor.keyspace,
  1001  			qo.getSelectLimit(),
  1002  			setVarComment,
  1003  			vcursor.safeSession.SystemVariables,
  1004  			vcursor,
  1005  		)
  1006  		if err != nil {
  1007  			return nil, nil, err
  1008  		}
  1009  		statement = result.AST
  1010  		bindVarNeeds = result.BindVarNeeds
  1011  		query = sqlparser.String(statement)
  1012  	}
  1013  
  1014  	logStats.SQL = comments.Leading + query + comments.Trailing
  1015  	logStats.BindVariables = sqltypes.CopyBindVariables(bindVars)
  1016  
  1017  	return e.cacheAndBuildStatement(ctx, vcursor, query, statement, qo, logStats, stmt, reservedVars, bindVarNeeds)
  1018  }
  1019  
  1020  func (e *Executor) cacheAndBuildStatement(ctx context.Context, vcursor *vcursorImpl, query string, statement sqlparser.Statement, qo iQueryOption, logStats *logstats.LogStats, stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, bindVarNeeds *sqlparser.BindVarNeeds) (*engine.Plan, sqlparser.Statement, error) {
  1021  	planHash := sha256.New()
  1022  	_, _ = planHash.Write([]byte(vcursor.planPrefixKey(ctx)))
  1023  	_, _ = planHash.Write([]byte{':'})
  1024  	_, _ = planHash.Write(hack.StringBytes(query))
  1025  	planKey := hex.EncodeToString(planHash.Sum(nil))
  1026  
  1027  	if sqlparser.CachePlan(statement) && qo.cachePlan() {
  1028  		if plan, ok := e.plans.Get(planKey); ok {
  1029  			logStats.CachedPlan = true
  1030  			return plan.(*engine.Plan), stmt, nil
  1031  		}
  1032  	}
  1033  
  1034  	plan, err := planbuilder.BuildFromStmt(query, statement, reservedVars, vcursor, bindVarNeeds, enableOnlineDDL, enableDirectDDL)
  1035  	if err != nil {
  1036  		return nil, nil, err
  1037  	}
  1038  
  1039  	plan.Warnings = vcursor.warnings
  1040  	vcursor.warnings = nil
  1041  
  1042  	err = e.checkThatPlanIsValid(stmt, plan)
  1043  	// Only cache the plan if it is valid (i.e. does not scatter)
  1044  	if err == nil && qo.cachePlan() && sqlparser.CachePlan(statement) {
  1045  		e.plans.Set(planKey, plan)
  1046  	}
  1047  	return plan, stmt, err
  1048  }
  1049  
  1050  func (e *Executor) canNormalizeStatement(stmt sqlparser.Statement, qo iQueryOption, setVarComment string) bool {
  1051  	return (e.normalize && sqlparser.CanNormalize(stmt)) ||
  1052  		sqlparser.MustRewriteAST(stmt, qo.getSelectLimit() > 0) || setVarComment != ""
  1053  }
  1054  
  1055  func prepareSetVarComment(vcursor *vcursorImpl, stmt sqlparser.Statement) (string, error) {
  1056  	if vcursor == nil || vcursor.Session().InReservedConn() {
  1057  		return "", nil
  1058  	}
  1059  
  1060  	if !vcursor.Session().HasSystemVariables() {
  1061  		return "", nil
  1062  	}
  1063  
  1064  	switch stmt.(type) {
  1065  	// If the statement is a transaction statement or a set no reserved connection / SET_VAR is needed
  1066  	case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback, *sqlparser.Savepoint,
  1067  		*sqlparser.SRollback, *sqlparser.Release, *sqlparser.Set, *sqlparser.Show:
  1068  		return "", nil
  1069  	case sqlparser.SupportOptimizerHint:
  1070  		break
  1071  	default:
  1072  		vcursor.NeedsReservedConn()
  1073  		return "", nil
  1074  	}
  1075  
  1076  	var res strings.Builder
  1077  	vcursor.Session().GetSystemVariables(func(k, v string) {
  1078  		res.WriteString(fmt.Sprintf("SET_VAR(%s = %s) ", k, v))
  1079  	})
  1080  	return strings.TrimSpace(res.String()), nil
  1081  }
  1082  
  1083  func (e *Executor) debugGetPlan(planKey string) (*engine.Plan, bool) {
  1084  	planHash := sha256.Sum256([]byte(planKey))
  1085  	planHex := hex.EncodeToString(planHash[:])
  1086  	if plan, ok := e.plans.Get(planHex); ok {
  1087  		return plan.(*engine.Plan), true
  1088  	}
  1089  	return nil, false
  1090  }
  1091  
  1092  type cacheItem struct {
  1093  	Key   string
  1094  	Value *engine.Plan
  1095  }
  1096  
  1097  func (e *Executor) debugCacheEntries() (items []cacheItem) {
  1098  	e.plans.ForEach(func(value any) bool {
  1099  		plan := value.(*engine.Plan)
  1100  		items = append(items, cacheItem{
  1101  			Key:   plan.Original,
  1102  			Value: plan,
  1103  		})
  1104  		return true
  1105  	})
  1106  	return
  1107  }
  1108  
  1109  // ServeHTTP shows the current plans in the query cache.
  1110  func (e *Executor) ServeHTTP(response http.ResponseWriter, request *http.Request) {
  1111  	if err := acl.CheckAccessHTTP(request, acl.DEBUGGING); err != nil {
  1112  		acl.SendError(response, err)
  1113  		return
  1114  	}
  1115  
  1116  	switch request.URL.Path {
  1117  	case pathQueryPlans:
  1118  		returnAsJSON(response, e.debugCacheEntries())
  1119  	case pathVSchema:
  1120  		returnAsJSON(response, e.VSchema())
  1121  	case pathScatterStats:
  1122  		e.WriteScatterStats(response)
  1123  	default:
  1124  		response.WriteHeader(http.StatusNotFound)
  1125  	}
  1126  }
  1127  
  1128  func returnAsJSON(response http.ResponseWriter, stuff any) {
  1129  	response.Header().Set("Content-Type", "application/json; charset=utf-8")
  1130  	buf, err := json.MarshalIndent(stuff, "", " ")
  1131  	if err != nil {
  1132  		_, _ = response.Write([]byte(err.Error()))
  1133  		return
  1134  	}
  1135  	ebuf := bytes.NewBuffer(nil)
  1136  	json.HTMLEscape(ebuf, buf)
  1137  	_, _ = response.Write(ebuf.Bytes())
  1138  }
  1139  
  1140  // Plans returns the LRU plan cache
  1141  func (e *Executor) Plans() cache.Cache {
  1142  	return e.plans
  1143  }
  1144  
  1145  func (e *Executor) updateQueryCounts(planType, keyspace, tableName string, shardQueries int64) {
  1146  	queriesProcessed.Add(planType, 1)
  1147  	queriesRouted.Add(planType, shardQueries)
  1148  	if tableName != "" {
  1149  		queriesProcessedByTable.Add([]string{planType, keyspace, tableName}, 1)
  1150  		queriesRoutedByTable.Add([]string{planType, keyspace, tableName}, shardQueries)
  1151  	}
  1152  }
  1153  
  1154  // VSchemaStats returns the loaded vschema stats.
  1155  func (e *Executor) VSchemaStats() *VSchemaStats {
  1156  	e.mu.Lock()
  1157  	defer e.mu.Unlock()
  1158  	if e.vschemaStats == nil {
  1159  		return &VSchemaStats{
  1160  			Error: "No VSchema loaded yet.",
  1161  		}
  1162  	}
  1163  	return e.vschemaStats
  1164  }
  1165  
  1166  func buildVarCharFields(names ...string) []*querypb.Field {
  1167  	fields := make([]*querypb.Field, len(names))
  1168  	for i, v := range names {
  1169  		fields[i] = &querypb.Field{
  1170  			Name:    v,
  1171  			Type:    sqltypes.VarChar,
  1172  			Charset: collations.CollationUtf8ID,
  1173  			Flags:   uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
  1174  		}
  1175  	}
  1176  	return fields
  1177  }
  1178  
  1179  func buildVarCharRow(values ...string) []sqltypes.Value {
  1180  	row := make([]sqltypes.Value, len(values))
  1181  	for i, v := range values {
  1182  		row[i] = sqltypes.NewVarChar(v)
  1183  	}
  1184  	return row
  1185  }
  1186  
  1187  // isValidPayloadSize validates whether a query payload is above the
  1188  // configured MaxPayloadSize threshold. The WarnPayloadSizeExceeded will increment
  1189  // if the payload size exceeds the warnPayloadSize.
  1190  
  1191  func isValidPayloadSize(query string) bool {
  1192  	payloadSize := len(query)
  1193  	if maxPayloadSize > 0 && payloadSize > maxPayloadSize {
  1194  		return false
  1195  	}
  1196  	if warnPayloadSize > 0 && payloadSize > warnPayloadSize {
  1197  		warnings.Add("WarnPayloadSizeExceeded", 1)
  1198  	}
  1199  	return true
  1200  }
  1201  
  1202  // Prepare executes a prepare statements.
  1203  func (e *Executor) Prepare(ctx context.Context, method string, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable) (fld []*querypb.Field, err error) {
  1204  	logStats := logstats.NewLogStats(ctx, method, sql, safeSession.GetSessionUUID(), bindVars)
  1205  	fld, err = e.prepare(ctx, safeSession, sql, bindVars, logStats)
  1206  	logStats.Error = err
  1207  
  1208  	// The mysql plugin runs an implicit rollback whenever a connection closes.
  1209  	// To avoid spamming the log with no-op rollback records, ignore it if
  1210  	// it was a no-op record (i.e. didn't issue any queries)
  1211  	if !(logStats.StmtType == "ROLLBACK" && logStats.ShardQueries == 0) {
  1212  		logStats.SaveEndTime()
  1213  		QueryLogger.Send(logStats)
  1214  	}
  1215  	return fld, err
  1216  }
  1217  
  1218  func (e *Executor) prepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) ([]*querypb.Field, error) {
  1219  	// Start an implicit transaction if necessary.
  1220  	if !safeSession.Autocommit && !safeSession.InTransaction() {
  1221  		if err := e.txConn.Begin(ctx, safeSession, nil); err != nil {
  1222  			return nil, err
  1223  		}
  1224  	}
  1225  
  1226  	if bindVars == nil {
  1227  		bindVars = make(map[string]*querypb.BindVariable)
  1228  	}
  1229  
  1230  	stmtType := sqlparser.Preview(sql)
  1231  	logStats.StmtType = stmtType.String()
  1232  
  1233  	// Mysql warnings are scoped to the current session, but are
  1234  	// cleared when a "non-diagnostic statement" is executed:
  1235  	// https://dev.mysql.com/doc/refman/8.0/en/show-warnings.html
  1236  	//
  1237  	// To emulate this behavior, clear warnings from the session
  1238  	// for all statements _except_ SHOW, so that SHOW WARNINGS
  1239  	// can actually return them.
  1240  	if stmtType != sqlparser.StmtShow {
  1241  		safeSession.ClearWarnings()
  1242  	}
  1243  
  1244  	switch stmtType {
  1245  	case sqlparser.StmtSelect, sqlparser.StmtShow:
  1246  		return e.handlePrepare(ctx, safeSession, sql, bindVars, logStats)
  1247  	case sqlparser.StmtDDL, sqlparser.StmtBegin, sqlparser.StmtCommit, sqlparser.StmtRollback, sqlparser.StmtSet, sqlparser.StmtInsert, sqlparser.StmtReplace, sqlparser.StmtUpdate, sqlparser.StmtDelete,
  1248  		sqlparser.StmtUse, sqlparser.StmtOther, sqlparser.StmtComment, sqlparser.StmtExplain, sqlparser.StmtFlush:
  1249  		return nil, nil
  1250  	}
  1251  	return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unrecognized prepare statement: %s", sql)
  1252  }
  1253  
  1254  func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) ([]*querypb.Field, error) {
  1255  	// V3 mode.
  1256  	query, comments := sqlparser.SplitMarginComments(sql)
  1257  	vcursor, _ := newVCursorImpl(safeSession, comments, e, logStats, e.vm, e.VSchema(), e.resolver.resolver, e.serv, e.warnShardedOnly, e.pv)
  1258  	plan, _, err := e.getPlan(ctx, vcursor, query, comments, bindVars, safeSession, logStats)
  1259  	execStart := time.Now()
  1260  	logStats.PlanTime = execStart.Sub(logStats.StartTime)
  1261  
  1262  	if err != nil {
  1263  		logStats.Error = err
  1264  		return nil, err
  1265  	}
  1266  
  1267  	err = e.addNeededBindVars(plan.BindVarNeeds, bindVars, safeSession)
  1268  	if err != nil {
  1269  		logStats.Error = err
  1270  		return nil, err
  1271  	}
  1272  
  1273  	qr, err := plan.Instructions.GetFields(ctx, vcursor, bindVars)
  1274  	logStats.ExecuteTime = time.Since(execStart)
  1275  	var errCount uint64
  1276  	if err != nil {
  1277  		logStats.Error = err
  1278  		errCount = 1 // nolint
  1279  		return nil, err
  1280  	}
  1281  	logStats.RowsAffected = qr.RowsAffected
  1282  
  1283  	plan.AddStats(1, time.Since(logStats.StartTime), logStats.ShardQueries, qr.RowsAffected, uint64(len(qr.Rows)), errCount)
  1284  
  1285  	return qr.Fields, err
  1286  }
  1287  
  1288  // ExecuteMultiShard implements the IExecutor interface
  1289  func (e *Executor) ExecuteMultiShard(ctx context.Context, primitive engine.Primitive, rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, session *SafeSession, autocommit bool, ignoreMaxMemoryRows bool) (qr *sqltypes.Result, errs []error) {
  1290  	return e.scatterConn.ExecuteMultiShard(ctx, primitive, rss, queries, session, autocommit, ignoreMaxMemoryRows)
  1291  }
  1292  
  1293  // StreamExecuteMulti implements the IExecutor interface
  1294  func (e *Executor) StreamExecuteMulti(ctx context.Context, primitive engine.Primitive, query string, rss []*srvtopo.ResolvedShard, vars []map[string]*querypb.BindVariable, session *SafeSession, autocommit bool, callback func(reply *sqltypes.Result) error) []error {
  1295  	return e.scatterConn.StreamExecuteMulti(ctx, primitive, query, rss, vars, session, autocommit, callback)
  1296  }
  1297  
  1298  // ExecuteLock implements the IExecutor interface
  1299  func (e *Executor) ExecuteLock(ctx context.Context, rs *srvtopo.ResolvedShard, query *querypb.BoundQuery, session *SafeSession, lockFuncType sqlparser.LockingFuncType) (*sqltypes.Result, error) {
  1300  	return e.scatterConn.ExecuteLock(ctx, rs, query, session, lockFuncType)
  1301  }
  1302  
  1303  // ExecuteMessageStream implements the IExecutor interface
  1304  func (e *Executor) ExecuteMessageStream(ctx context.Context, rss []*srvtopo.ResolvedShard, tableName string, callback func(reply *sqltypes.Result) error) error {
  1305  	return e.scatterConn.MessageStream(ctx, rss, tableName, callback)
  1306  }
  1307  
  1308  // ExecuteVStream implements the IExecutor interface
  1309  func (e *Executor) ExecuteVStream(ctx context.Context, rss []*srvtopo.ResolvedShard, filter *binlogdatapb.Filter, gtid string, callback func(evs []*binlogdatapb.VEvent) error) error {
  1310  	return e.startVStream(ctx, rss, filter, gtid, callback)
  1311  }
  1312  
  1313  func (e *Executor) startVStream(ctx context.Context, rss []*srvtopo.ResolvedShard, filter *binlogdatapb.Filter, gtid string, callback func(evs []*binlogdatapb.VEvent) error) error {
  1314  	var shardGtids []*binlogdatapb.ShardGtid
  1315  	for _, rs := range rss {
  1316  		shardGtid := &binlogdatapb.ShardGtid{
  1317  			Keyspace: rs.Target.Keyspace,
  1318  			Shard:    rs.Target.Shard,
  1319  			Gtid:     gtid,
  1320  		}
  1321  		shardGtids = append(shardGtids, shardGtid)
  1322  	}
  1323  	vgtid := &binlogdatapb.VGtid{
  1324  		ShardGtids: shardGtids,
  1325  	}
  1326  	ts, err := e.serv.GetTopoServer()
  1327  	if err != nil {
  1328  		return err
  1329  	}
  1330  
  1331  	vsm := newVStreamManager(e.resolver.resolver, e.serv, e.cell)
  1332  	vs := &vstream{
  1333  		vgtid:              vgtid,
  1334  		tabletType:         topodatapb.TabletType_PRIMARY,
  1335  		filter:             filter,
  1336  		send:               callback,
  1337  		resolver:           e.resolver.resolver,
  1338  		journaler:          make(map[int64]*journalEvent),
  1339  		skewTimeoutSeconds: maxSkewTimeoutSeconds,
  1340  		timestamps:         make(map[string]int64),
  1341  		vsm:                vsm,
  1342  		eventCh:            make(chan []*binlogdatapb.VEvent),
  1343  		ts:                 ts,
  1344  		copyCompletedShard: make(map[string]struct{}),
  1345  	}
  1346  	_ = vs.stream(ctx)
  1347  	return nil
  1348  }
  1349  
  1350  func (e *Executor) checkThatPlanIsValid(stmt sqlparser.Statement, plan *engine.Plan) error {
  1351  	if e.allowScatter || plan.Instructions == nil || sqlparser.AllowScatterDirective(stmt) {
  1352  		return nil
  1353  	}
  1354  	// we go over all the primitives in the plan, searching for a route that is of SelectScatter opcode
  1355  	badPrimitive := engine.Find(func(node engine.Primitive) bool {
  1356  		router, ok := node.(*engine.Route)
  1357  		if !ok {
  1358  			return false
  1359  		}
  1360  		return router.Opcode == engine.Scatter
  1361  	}, plan.Instructions)
  1362  
  1363  	if badPrimitive == nil {
  1364  		return nil
  1365  	}
  1366  
  1367  	return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "plan includes scatter, which is disallowed using the `no_scatter` command line argument")
  1368  }
  1369  
  1370  func getTabletThrottlerStatus(tabletHostPort string) (string, error) {
  1371  	client := http.Client{
  1372  		Timeout: 100 * time.Millisecond,
  1373  	}
  1374  	resp, err := client.Get(fmt.Sprintf("http://%s/throttler/check?app=vtgate", tabletHostPort))
  1375  	if err != nil {
  1376  		return "", err
  1377  	}
  1378  	defer resp.Body.Close()
  1379  	body, err := io.ReadAll(resp.Body)
  1380  	if err != nil {
  1381  		return "", err
  1382  	}
  1383  
  1384  	var elements struct {
  1385  		StatusCode int
  1386  		Value      float64
  1387  		Threshold  float64
  1388  		Message    string
  1389  	}
  1390  	err = json.Unmarshal(body, &elements)
  1391  	if err != nil {
  1392  		return "", err
  1393  	}
  1394  
  1395  	httpStatusStr := http.StatusText(elements.StatusCode)
  1396  
  1397  	load := float64(0)
  1398  	if elements.Threshold > 0 {
  1399  		load = float64((elements.Value / elements.Threshold) * 100)
  1400  	}
  1401  
  1402  	status := fmt.Sprintf("{\"state\":\"%s\",\"load\":%.2f,\"message\":\"%s\"}", httpStatusStr, load, elements.Message)
  1403  	return status, nil
  1404  }
  1405  
  1406  // ReleaseLock implements the IExecutor interface
  1407  func (e *Executor) ReleaseLock(ctx context.Context, session *SafeSession) error {
  1408  	return e.txConn.ReleaseLock(ctx, session)
  1409  }