vitess.io/vitess@v0.16.2/go/vt/vtgate/executor.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vtgate 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/sha256" 23 "encoding/hex" 24 "encoding/json" 25 "errors" 26 "fmt" 27 "io" 28 "net/http" 29 "strings" 30 "sync" 31 "time" 32 33 "github.com/spf13/pflag" 34 35 "vitess.io/vitess/go/acl" 36 "vitess.io/vitess/go/cache" 37 "vitess.io/vitess/go/hack" 38 "vitess.io/vitess/go/mysql/collations" 39 "vitess.io/vitess/go/sqltypes" 40 "vitess.io/vitess/go/stats" 41 "vitess.io/vitess/go/sync2" 42 "vitess.io/vitess/go/trace" 43 "vitess.io/vitess/go/vt/callerid" 44 "vitess.io/vitess/go/vt/key" 45 "vitess.io/vitess/go/vt/log" 46 "vitess.io/vitess/go/vt/servenv" 47 "vitess.io/vitess/go/vt/sqlparser" 48 "vitess.io/vitess/go/vt/srvtopo" 49 "vitess.io/vitess/go/vt/sysvars" 50 "vitess.io/vitess/go/vt/topo/topoproto" 51 "vitess.io/vitess/go/vt/vterrors" 52 "vitess.io/vitess/go/vt/vtgate/engine" 53 "vitess.io/vitess/go/vt/vtgate/evalengine" 54 "vitess.io/vitess/go/vt/vtgate/logstats" 55 "vitess.io/vitess/go/vt/vtgate/planbuilder" 56 "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" 57 "vitess.io/vitess/go/vt/vtgate/vindexes" 58 "vitess.io/vitess/go/vt/vtgate/vschemaacl" 59 60 binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" 61 querypb "vitess.io/vitess/go/vt/proto/query" 62 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 63 vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" 64 vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" 65 ) 66 67 var ( 68 errNoKeyspace = vterrors.VT09005() 69 defaultTabletType = topodatapb.TabletType_PRIMARY 70 71 // TODO: @rafael - These two counters should be deprecated in favor of the ByTable ones in v17+. They are kept for now for backwards compatibility. 72 queriesProcessed = stats.NewCountersWithSingleLabel("QueriesProcessed", "Queries processed at vtgate by plan type", "Plan") 73 queriesRouted = stats.NewCountersWithSingleLabel("QueriesRouted", "Queries routed from vtgate to vttablet by plan type", "Plan") 74 75 queriesProcessedByTable = stats.NewCountersWithMultiLabels("QueriesProcessedByTable", "Queries processed at vtgate by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"}) 76 queriesRoutedByTable = stats.NewCountersWithMultiLabels("QueriesRoutedByTable", "Queries routed from vtgate to vttablet by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"}) 77 ) 78 79 const ( 80 bindVarPrefix = "__vt" 81 ) 82 83 func init() { 84 registerTabletTypeFlag := func(fs *pflag.FlagSet) { 85 fs.Var((*topoproto.TabletTypeFlag)(&defaultTabletType), "default_tablet_type", "The default tablet type to set for queries, when one is not explicitly selected.") 86 } 87 88 servenv.OnParseFor("vtgate", registerTabletTypeFlag) 89 servenv.OnParseFor("vtgateclienttest", registerTabletTypeFlag) 90 servenv.OnParseFor("vtcombo", registerTabletTypeFlag) 91 servenv.OnParseFor("vtexplain", registerTabletTypeFlag) 92 } 93 94 // Executor is the engine that executes queries by utilizing 95 // the abilities of the underlying vttablets. 96 type Executor struct { 97 serv srvtopo.Server 98 cell string 99 resolver *Resolver 100 scatterConn *ScatterConn 101 txConn *TxConn 102 pv plancontext.PlannerVersion 103 104 mu sync.Mutex 105 vschema *vindexes.VSchema 106 streamSize int 107 plans cache.Cache 108 vschemaStats *VSchemaStats 109 110 normalize bool 111 warnShardedOnly bool 112 113 vm *VSchemaManager 114 schemaTracker SchemaInfo 115 116 // allowScatter will fail planning if set to false and a plan contains any scatter queries 117 allowScatter bool 118 } 119 120 var executorOnce sync.Once 121 122 const pathQueryPlans = "/debug/query_plans" 123 const pathScatterStats = "/debug/scatter_stats" 124 const pathVSchema = "/debug/vschema" 125 126 // NewExecutor creates a new Executor. 127 func NewExecutor( 128 ctx context.Context, 129 serv srvtopo.Server, 130 cell string, 131 resolver *Resolver, 132 normalize, warnOnShardedOnly bool, 133 streamSize int, 134 cacheCfg *cache.Config, 135 schemaTracker SchemaInfo, 136 noScatter bool, 137 pv plancontext.PlannerVersion, 138 ) *Executor { 139 e := &Executor{ 140 serv: serv, 141 cell: cell, 142 resolver: resolver, 143 scatterConn: resolver.scatterConn, 144 txConn: resolver.scatterConn.txConn, 145 plans: cache.NewDefaultCacheImpl(cacheCfg), 146 normalize: normalize, 147 warnShardedOnly: warnOnShardedOnly, 148 streamSize: streamSize, 149 schemaTracker: schemaTracker, 150 allowScatter: !noScatter, 151 pv: pv, 152 } 153 154 vschemaacl.Init() 155 // we subscribe to update from the VSchemaManager 156 e.vm = &VSchemaManager{ 157 subscriber: e.SaveVSchema, 158 serv: serv, 159 cell: cell, 160 schema: e.schemaTracker, 161 } 162 serv.WatchSrvVSchema(ctx, cell, e.vm.VSchemaUpdate) 163 164 executorOnce.Do(func() { 165 stats.NewGaugeFunc("QueryPlanCacheLength", "Query plan cache length", func() int64 { 166 return int64(e.plans.Len()) 167 }) 168 stats.NewGaugeFunc("QueryPlanCacheSize", "Query plan cache size", func() int64 { 169 return e.plans.UsedCapacity() 170 }) 171 stats.NewGaugeFunc("QueryPlanCacheCapacity", "Query plan cache capacity", func() int64 { 172 return e.plans.MaxCapacity() 173 }) 174 stats.NewCounterFunc("QueryPlanCacheEvictions", "Query plan cache evictions", func() int64 { 175 return e.plans.Evictions() 176 }) 177 stats.NewCounterFunc("QueryPlanCacheHits", "Query plan cache hits", func() int64 { 178 return e.plans.Hits() 179 }) 180 stats.NewCounterFunc("QueryPlanCacheMisses", "Query plan cache misses", func() int64 { 181 return e.plans.Misses() 182 }) 183 http.Handle(pathQueryPlans, e) 184 http.Handle(pathScatterStats, e) 185 http.Handle(pathVSchema, e) 186 }) 187 return e 188 } 189 190 // Execute executes a non-streaming query. 191 func (e *Executor) Execute(ctx context.Context, method string, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable) (result *sqltypes.Result, err error) { 192 span, ctx := trace.NewSpan(ctx, "executor.Execute") 193 span.Annotate("method", method) 194 trace.AnnotateSQL(span, sqlparser.Preview(sql)) 195 defer span.Finish() 196 197 logStats := logstats.NewLogStats(ctx, method, sql, safeSession.GetSessionUUID(), bindVars) 198 stmtType, result, err := e.execute(ctx, safeSession, sql, bindVars, logStats) 199 logStats.Error = err 200 if result == nil { 201 saveSessionStats(safeSession, stmtType, 0, 0, 0, err) 202 } else { 203 saveSessionStats(safeSession, stmtType, result.RowsAffected, result.InsertID, len(result.Rows), err) 204 } 205 if result != nil && len(result.Rows) > warnMemoryRows { 206 warnings.Add("ResultsExceeded", 1) 207 piiSafeSQL, err := sqlparser.RedactSQLQuery(sql) 208 if err != nil { 209 piiSafeSQL = logStats.StmtType 210 } 211 log.Warningf("%q exceeds warning threshold of max memory rows: %v", piiSafeSQL, warnMemoryRows) 212 } 213 214 logStats.SaveEndTime() 215 QueryLogger.Send(logStats) 216 return result, err 217 } 218 219 type streaminResultReceiver struct { 220 mu sync.Mutex 221 stmtType sqlparser.StatementType 222 rowsAffected uint64 223 rowsReturned int 224 insertID uint64 225 callback func(*sqltypes.Result) error 226 } 227 228 func (s *streaminResultReceiver) storeResultStats(typ sqlparser.StatementType, qr *sqltypes.Result) error { 229 s.mu.Lock() 230 defer s.mu.Unlock() 231 s.rowsAffected += qr.RowsAffected 232 s.rowsReturned += len(qr.Rows) 233 if qr.InsertID != 0 { 234 s.insertID = qr.InsertID 235 } 236 s.stmtType = typ 237 return s.callback(qr) 238 } 239 240 // StreamExecute executes a streaming query. 241 func (e *Executor) StreamExecute( 242 ctx context.Context, 243 method string, 244 safeSession *SafeSession, 245 sql string, 246 bindVars map[string]*querypb.BindVariable, 247 callback func(*sqltypes.Result) error, 248 ) error { 249 span, ctx := trace.NewSpan(ctx, "executor.StreamExecute") 250 span.Annotate("method", method) 251 trace.AnnotateSQL(span, sqlparser.Preview(sql)) 252 defer span.Finish() 253 254 logStats := logstats.NewLogStats(ctx, method, sql, safeSession.GetSessionUUID(), bindVars) 255 srr := &streaminResultReceiver{callback: callback} 256 var err error 257 258 resultHandler := func(ctx context.Context, plan *engine.Plan, vc *vcursorImpl, bindVars map[string]*querypb.BindVariable, execStart time.Time) error { 259 var seenResults sync2.AtomicBool 260 var resultMu sync.Mutex 261 result := &sqltypes.Result{} 262 if canReturnRows(plan.Type) { 263 srr.callback = func(qr *sqltypes.Result) error { 264 resultMu.Lock() 265 defer resultMu.Unlock() 266 // If the row has field info, send it separately. 267 // TODO(sougou): this behavior is for handling tests because 268 // the framework currently sends all results as one packet. 269 byteCount := 0 270 if len(qr.Fields) > 0 { 271 if err := callback(qr.Metadata()); err != nil { 272 return err 273 } 274 seenResults.Set(true) 275 } 276 277 for _, row := range qr.Rows { 278 result.Rows = append(result.Rows, row) 279 280 for _, col := range row { 281 byteCount += col.Len() 282 } 283 284 if byteCount >= e.streamSize { 285 err := callback(result) 286 seenResults.Set(true) 287 result = &sqltypes.Result{} 288 byteCount = 0 289 if err != nil { 290 return err 291 } 292 } 293 } 294 return nil 295 } 296 } 297 298 // 4: Execute! 299 err := vc.StreamExecutePrimitive(ctx, plan.Instructions, bindVars, true, func(qr *sqltypes.Result) error { 300 return srr.storeResultStats(plan.Type, qr) 301 }) 302 303 // Check if there was partial DML execution. If so, rollback the effect of the partially executed query. 304 if err != nil { 305 if !canReturnRows(plan.Type) { 306 return e.rollbackExecIfNeeded(ctx, safeSession, bindVars, logStats, err) 307 } 308 return err 309 } 310 311 if !canReturnRows(plan.Type) { 312 return nil 313 } 314 315 // Send left-over rows if there is no error on execution. 316 if len(result.Rows) > 0 || !seenResults.Get() { 317 if err := callback(result); err != nil { 318 return err 319 } 320 } 321 322 // 5: Log and add statistics 323 logStats.TablesUsed = plan.TablesUsed 324 logStats.TabletType = vc.TabletType().String() 325 logStats.ExecuteTime = time.Since(execStart) 326 logStats.ActiveKeyspace = vc.keyspace 327 328 e.updateQueryCounts(plan.Instructions.RouteType(), plan.Instructions.GetKeyspaceName(), plan.Instructions.GetTableName(), int64(logStats.ShardQueries)) 329 330 return err 331 } 332 333 err = e.newExecute(ctx, safeSession, sql, bindVars, logStats, resultHandler, srr.storeResultStats) 334 335 logStats.Error = err 336 saveSessionStats(safeSession, srr.stmtType, srr.rowsAffected, srr.insertID, srr.rowsReturned, err) 337 if srr.rowsReturned > warnMemoryRows { 338 warnings.Add("ResultsExceeded", 1) 339 piiSafeSQL, err := sqlparser.RedactSQLQuery(sql) 340 if err != nil { 341 piiSafeSQL = logStats.StmtType 342 } 343 log.Warningf("%q exceeds warning threshold of max memory rows: %v", piiSafeSQL, warnMemoryRows) 344 } 345 346 logStats.SaveEndTime() 347 QueryLogger.Send(logStats) 348 return err 349 350 } 351 352 func canReturnRows(stmtType sqlparser.StatementType) bool { 353 switch stmtType { 354 case sqlparser.StmtSelect, sqlparser.StmtShow, sqlparser.StmtExplain, sqlparser.StmtCallProc: 355 return true 356 default: 357 return false 358 } 359 } 360 361 func saveSessionStats(safeSession *SafeSession, stmtType sqlparser.StatementType, rowsAffected, insertID uint64, rowsReturned int, err error) { 362 safeSession.RowCount = -1 363 if err != nil { 364 return 365 } 366 if !safeSession.foundRowsHandled { 367 safeSession.FoundRows = uint64(rowsReturned) 368 } 369 if insertID > 0 { 370 safeSession.LastInsertId = insertID 371 } 372 switch stmtType { 373 case sqlparser.StmtInsert, sqlparser.StmtReplace, sqlparser.StmtUpdate, sqlparser.StmtDelete: 374 safeSession.RowCount = int64(rowsAffected) 375 case sqlparser.StmtDDL, sqlparser.StmtSet, sqlparser.StmtBegin, sqlparser.StmtCommit, sqlparser.StmtRollback, sqlparser.StmtFlush: 376 safeSession.RowCount = 0 377 } 378 } 379 380 func (e *Executor) execute(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) (sqlparser.StatementType, *sqltypes.Result, error) { 381 var err error 382 var qr *sqltypes.Result 383 var stmtType sqlparser.StatementType 384 err = e.newExecute(ctx, safeSession, sql, bindVars, logStats, func(ctx context.Context, plan *engine.Plan, vc *vcursorImpl, bindVars map[string]*querypb.BindVariable, time time.Time) error { 385 stmtType = plan.Type 386 qr, err = e.executePlan(ctx, safeSession, plan, vc, bindVars, logStats, time) 387 return err 388 }, func(typ sqlparser.StatementType, result *sqltypes.Result) error { 389 stmtType = typ 390 qr = result 391 return nil 392 }) 393 394 return stmtType, qr, err 395 } 396 397 // addNeededBindVars adds bind vars that are needed by the plan 398 func (e *Executor) addNeededBindVars(bindVarNeeds *sqlparser.BindVarNeeds, bindVars map[string]*querypb.BindVariable, session *SafeSession) error { 399 for _, funcName := range bindVarNeeds.NeedFunctionResult { 400 switch funcName { 401 case sqlparser.DBVarName: 402 bindVars[sqlparser.DBVarName] = sqltypes.StringBindVariable(session.TargetString) 403 case sqlparser.LastInsertIDName: 404 bindVars[sqlparser.LastInsertIDName] = sqltypes.Uint64BindVariable(session.GetLastInsertId()) 405 case sqlparser.FoundRowsName: 406 bindVars[sqlparser.FoundRowsName] = sqltypes.Int64BindVariable(int64(session.FoundRows)) 407 case sqlparser.RowCountName: 408 bindVars[sqlparser.RowCountName] = sqltypes.Int64BindVariable(session.RowCount) 409 } 410 } 411 412 for _, sysVar := range bindVarNeeds.NeedSystemVariable { 413 key := bindVarPrefix + sysVar 414 switch sysVar { 415 case sysvars.Autocommit.Name: 416 bindVars[key] = sqltypes.BoolBindVariable(session.Autocommit) 417 case sysvars.QueryTimeout.Name: 418 bindVars[key] = sqltypes.Int64BindVariable(session.GetQueryTimeout()) 419 case sysvars.ClientFoundRows.Name: 420 var v bool 421 ifOptionsExist(session, func(options *querypb.ExecuteOptions) { 422 v = options.ClientFoundRows 423 }) 424 bindVars[key] = sqltypes.BoolBindVariable(v) 425 case sysvars.SkipQueryPlanCache.Name: 426 var v bool 427 ifOptionsExist(session, func(options *querypb.ExecuteOptions) { 428 v = options.ClientFoundRows 429 }) 430 bindVars[key] = sqltypes.BoolBindVariable(v) 431 case sysvars.SQLSelectLimit.Name: 432 var v int64 433 ifOptionsExist(session, func(options *querypb.ExecuteOptions) { 434 v = options.SqlSelectLimit 435 }) 436 bindVars[key] = sqltypes.Int64BindVariable(v) 437 case sysvars.TransactionMode.Name: 438 bindVars[key] = sqltypes.StringBindVariable(session.TransactionMode.String()) 439 case sysvars.Workload.Name: 440 var v string 441 ifOptionsExist(session, func(options *querypb.ExecuteOptions) { 442 v = options.GetWorkload().String() 443 }) 444 bindVars[key] = sqltypes.StringBindVariable(v) 445 case sysvars.DDLStrategy.Name: 446 bindVars[key] = sqltypes.StringBindVariable(session.DDLStrategy) 447 case sysvars.SessionUUID.Name: 448 bindVars[key] = sqltypes.StringBindVariable(session.SessionUUID) 449 case sysvars.SessionEnableSystemSettings.Name: 450 bindVars[key] = sqltypes.BoolBindVariable(session.EnableSystemSettings) 451 case sysvars.ReadAfterWriteGTID.Name: 452 var v string 453 ifReadAfterWriteExist(session, func(raw *vtgatepb.ReadAfterWrite) { 454 v = raw.ReadAfterWriteGtid 455 }) 456 bindVars[key] = sqltypes.StringBindVariable(v) 457 case sysvars.ReadAfterWriteTimeOut.Name: 458 var v float64 459 ifReadAfterWriteExist(session, func(raw *vtgatepb.ReadAfterWrite) { 460 v = raw.ReadAfterWriteTimeout 461 }) 462 bindVars[key] = sqltypes.Float64BindVariable(v) 463 case sysvars.SessionTrackGTIDs.Name: 464 v := "off" 465 ifReadAfterWriteExist(session, func(raw *vtgatepb.ReadAfterWrite) { 466 if raw.SessionTrackGtids { 467 v = "own_gtid" 468 } 469 }) 470 bindVars[key] = sqltypes.StringBindVariable(v) 471 case sysvars.Version.Name: 472 bindVars[key] = sqltypes.StringBindVariable(servenv.AppVersion.MySQLVersion()) 473 case sysvars.VersionComment.Name: 474 bindVars[key] = sqltypes.StringBindVariable(servenv.AppVersion.String()) 475 case sysvars.Socket.Name: 476 bindVars[key] = sqltypes.StringBindVariable(mysqlSocketPath()) 477 default: 478 if value, hasSysVar := session.SystemVariables[sysVar]; hasSysVar { 479 expr, err := sqlparser.ParseExpr(value) 480 if err != nil { 481 return err 482 } 483 484 evalExpr, err := evalengine.Translate(expr, nil) 485 if err != nil { 486 return err 487 } 488 evaluated, err := evalengine.EmptyExpressionEnv().Evaluate(evalExpr) 489 if err != nil { 490 return err 491 } 492 bindVars[key] = sqltypes.ValueBindVariable(evaluated.Value()) 493 } 494 } 495 } 496 497 udvMap := session.UserDefinedVariables 498 if udvMap == nil { 499 udvMap = map[string]*querypb.BindVariable{} 500 } 501 for _, udv := range bindVarNeeds.NeedUserDefinedVariables { 502 val := udvMap[udv] 503 if val == nil { 504 val = sqltypes.NullBindVariable 505 } 506 bindVars[sqlparser.UserDefinedVariableName+udv] = val 507 } 508 509 return nil 510 } 511 512 func ifOptionsExist(session *SafeSession, f func(*querypb.ExecuteOptions)) { 513 options := session.GetOptions() 514 if options != nil { 515 f(options) 516 } 517 } 518 519 func ifReadAfterWriteExist(session *SafeSession, f func(*vtgatepb.ReadAfterWrite)) { 520 raw := session.ReadAfterWrite 521 if raw != nil { 522 f(raw) 523 } 524 } 525 526 func (e *Executor) handleBegin(ctx context.Context, safeSession *SafeSession, logStats *logstats.LogStats, stmt sqlparser.Statement) (*sqltypes.Result, error) { 527 execStart := time.Now() 528 logStats.PlanTime = execStart.Sub(logStats.StartTime) 529 530 begin := stmt.(*sqlparser.Begin) 531 err := e.txConn.Begin(ctx, safeSession, begin.TxAccessModes) 532 logStats.ExecuteTime = time.Since(execStart) 533 534 e.updateQueryCounts("Begin", "", "", 0) 535 536 return &sqltypes.Result{}, err 537 } 538 539 func (e *Executor) handleCommit(ctx context.Context, safeSession *SafeSession, logStats *logstats.LogStats) (*sqltypes.Result, error) { 540 execStart := time.Now() 541 logStats.PlanTime = execStart.Sub(logStats.StartTime) 542 logStats.ShardQueries = uint64(len(safeSession.ShardSessions)) 543 e.updateQueryCounts("Commit", "", "", int64(logStats.ShardQueries)) 544 545 err := e.txConn.Commit(ctx, safeSession) 546 logStats.CommitTime = time.Since(execStart) 547 return &sqltypes.Result{}, err 548 } 549 550 // Commit commits the existing transactions 551 func (e *Executor) Commit(ctx context.Context, safeSession *SafeSession) error { 552 return e.txConn.Commit(ctx, safeSession) 553 } 554 555 func (e *Executor) handleRollback(ctx context.Context, safeSession *SafeSession, logStats *logstats.LogStats) (*sqltypes.Result, error) { 556 execStart := time.Now() 557 logStats.PlanTime = execStart.Sub(logStats.StartTime) 558 logStats.ShardQueries = uint64(len(safeSession.ShardSessions)) 559 e.updateQueryCounts("Rollback", "", "", int64(logStats.ShardQueries)) 560 err := e.txConn.Rollback(ctx, safeSession) 561 logStats.CommitTime = time.Since(execStart) 562 return &sqltypes.Result{}, err 563 } 564 565 func (e *Executor) handleSavepoint(ctx context.Context, safeSession *SafeSession, sql string, planType string, logStats *logstats.LogStats, nonTxResponse func(query string) (*sqltypes.Result, error), ignoreMaxMemoryRows bool) (*sqltypes.Result, error) { 566 execStart := time.Now() 567 logStats.PlanTime = execStart.Sub(logStats.StartTime) 568 logStats.ShardQueries = uint64(len(safeSession.ShardSessions)) 569 e.updateQueryCounts(planType, "", "", int64(logStats.ShardQueries)) 570 defer func() { 571 logStats.ExecuteTime = time.Since(execStart) 572 }() 573 574 // If no transaction exists on any of the shard sessions, 575 // then savepoint does not need to be executed, it will be only stored in the session 576 // and later will be executed when a transaction is started. 577 if !safeSession.isTxOpen() { 578 if safeSession.InTransaction() { 579 // Storing, as this needs to be executed just after starting transaction on the shard. 580 safeSession.StoreSavepoint(sql) 581 return &sqltypes.Result{}, nil 582 } 583 return nonTxResponse(sql) 584 } 585 orig := safeSession.commitOrder 586 qr, err := e.executeSPInAllSessions(ctx, safeSession, sql, ignoreMaxMemoryRows) 587 safeSession.SetCommitOrder(orig) 588 if err != nil { 589 return nil, err 590 } 591 safeSession.StoreSavepoint(sql) 592 return qr, nil 593 } 594 595 // executeSPInAllSessions function executes the savepoint query in all open shard sessions (pre, normal and post) 596 // which has non-zero transaction id (i.e. an open transaction on the shard connection). 597 func (e *Executor) executeSPInAllSessions(ctx context.Context, safeSession *SafeSession, sql string, ignoreMaxMemoryRows bool) (*sqltypes.Result, error) { 598 var qr *sqltypes.Result 599 var errs []error 600 for _, co := range []vtgatepb.CommitOrder{vtgatepb.CommitOrder_PRE, vtgatepb.CommitOrder_NORMAL, vtgatepb.CommitOrder_POST} { 601 safeSession.SetCommitOrder(co) 602 603 var rss []*srvtopo.ResolvedShard 604 var queries []*querypb.BoundQuery 605 for _, shardSession := range safeSession.getSessions() { 606 // This will avoid executing savepoint on reserved connections 607 // which has no open transaction. 608 if shardSession.TransactionId == 0 { 609 continue 610 } 611 rss = append(rss, &srvtopo.ResolvedShard{ 612 Target: shardSession.Target, 613 Gateway: e.resolver.resolver.GetGateway(), 614 }) 615 queries = append(queries, &querypb.BoundQuery{Sql: sql}) 616 } 617 qr, errs = e.ExecuteMultiShard(ctx, nil, rss, queries, safeSession, false /*autocommit*/, ignoreMaxMemoryRows) 618 err := vterrors.Aggregate(errs) 619 if err != nil { 620 return nil, err 621 } 622 } 623 return qr, nil 624 } 625 626 // CloseSession releases the current connection, which rollbacks open transactions and closes reserved connections. 627 // It is called then the MySQL servers closes the connection to its client. 628 func (e *Executor) CloseSession(ctx context.Context, safeSession *SafeSession) error { 629 return e.txConn.ReleaseAll(ctx, safeSession) 630 } 631 632 func (e *Executor) setVitessMetadata(ctx context.Context, name, value string) error { 633 // TODO(kalfonso): move to its own acl check and consolidate into an acl component that can handle multiple operations (vschema, metadata) 634 user := callerid.ImmediateCallerIDFromContext(ctx) 635 allowed := vschemaacl.Authorized(user) 636 if !allowed { 637 return vterrors.NewErrorf(vtrpcpb.Code_PERMISSION_DENIED, vterrors.AccessDeniedError, "User '%s' not authorized to perform vitess metadata operations", user.GetUsername()) 638 } 639 640 ts, err := e.serv.GetTopoServer() 641 if err != nil { 642 return err 643 } 644 645 if value == "" { 646 return ts.DeleteMetadata(ctx, name) 647 } 648 return ts.UpsertMetadata(ctx, name, value) 649 } 650 651 func (e *Executor) showVitessMetadata(ctx context.Context, filter *sqlparser.ShowFilter) (*sqltypes.Result, error) { 652 ts, err := e.serv.GetTopoServer() 653 if err != nil { 654 return nil, err 655 } 656 657 var metadata map[string]string 658 if filter == nil { 659 metadata, err = ts.GetMetadata(ctx, "") 660 if err != nil { 661 return nil, err 662 } 663 } else { 664 metadata, err = ts.GetMetadata(ctx, filter.Like) 665 if err != nil { 666 return nil, err 667 } 668 } 669 670 rows := make([][]sqltypes.Value, 0, len(metadata)) 671 for k, v := range metadata { 672 row := buildVarCharRow(k, v) 673 rows = append(rows, row) 674 } 675 676 return &sqltypes.Result{ 677 Fields: buildVarCharFields("Key", "Value"), 678 Rows: rows, 679 }, nil 680 } 681 682 type tabletFilter func(tablet *topodatapb.Tablet, servingState string, primaryTermStartTime int64) bool 683 684 func (e *Executor) showShards(ctx context.Context, filter *sqlparser.ShowFilter, destTabletType topodatapb.TabletType) (*sqltypes.Result, error) { 685 showVitessShardsFilters := func(filter *sqlparser.ShowFilter) ([]func(string) bool, []func(string, *topodatapb.ShardReference) bool) { 686 keyspaceFilters := []func(string) bool{} 687 shardFilters := []func(string, *topodatapb.ShardReference) bool{} 688 689 if filter == nil { 690 return keyspaceFilters, shardFilters 691 } 692 693 if filter.Like != "" { 694 shardLikeRexep := sqlparser.LikeToRegexp(filter.Like) 695 696 if strings.Contains(filter.Like, "/") { 697 keyspaceLikeRexep := sqlparser.LikeToRegexp(strings.Split(filter.Like, "/")[0]) 698 keyspaceFilters = append(keyspaceFilters, func(ks string) bool { 699 return keyspaceLikeRexep.MatchString(ks) 700 }) 701 } 702 shardFilters = append(shardFilters, func(ks string, shard *topodatapb.ShardReference) bool { 703 return shardLikeRexep.MatchString(topoproto.KeyspaceShardString(ks, shard.Name)) 704 }) 705 706 return keyspaceFilters, shardFilters 707 } 708 709 if filter.Filter != nil { 710 // TODO build a query planner I guess? lol that should be fun 711 log.Infof("SHOW VITESS_SHARDS where clause %+v. Ignoring this (for now).", filter.Filter) 712 } 713 714 return keyspaceFilters, shardFilters 715 } 716 717 keyspaceFilters, shardFilters := showVitessShardsFilters(filter) 718 719 keyspaces, err := e.resolver.resolver.GetAllKeyspaces(ctx) 720 if err != nil { 721 return nil, err 722 } 723 724 var rows [][]sqltypes.Value 725 for _, keyspace := range keyspaces { 726 skipKeyspace := false 727 for _, filter := range keyspaceFilters { 728 if !filter(keyspace) { 729 skipKeyspace = true 730 break 731 } 732 } 733 734 if skipKeyspace { 735 continue 736 } 737 738 _, _, shards, err := e.resolver.resolver.GetKeyspaceShards(ctx, keyspace, destTabletType) 739 if err != nil { 740 // There might be a misconfigured keyspace or no shards in the keyspace. 741 // Skip any errors and move on. 742 continue 743 } 744 745 for _, shard := range shards { 746 skipShard := false 747 for _, filter := range shardFilters { 748 if !filter(keyspace, shard) { 749 skipShard = true 750 break 751 } 752 } 753 754 if skipShard { 755 continue 756 } 757 758 rows = append(rows, buildVarCharRow(topoproto.KeyspaceShardString(keyspace, shard.Name))) 759 } 760 } 761 762 return &sqltypes.Result{ 763 Fields: buildVarCharFields("Shards"), 764 Rows: rows, 765 }, nil 766 } 767 768 func (e *Executor) showTablets(filter *sqlparser.ShowFilter) (*sqltypes.Result, error) { 769 getTabletFilters := func(filter *sqlparser.ShowFilter) []tabletFilter { 770 var filters []tabletFilter 771 772 if filter == nil { 773 return filters 774 } 775 776 if filter.Like != "" { 777 tabletRegexp := sqlparser.LikeToRegexp(filter.Like) 778 779 f := func(tablet *topodatapb.Tablet, servingState string, primaryTermStartTime int64) bool { 780 return tabletRegexp.MatchString(tablet.Hostname) 781 } 782 783 filters = append(filters, f) 784 return filters 785 } 786 787 if filter.Filter != nil { 788 log.Infof("SHOW VITESS_TABLETS where clause: %+v. Ignoring this (for now).", filter.Filter) 789 } 790 791 return filters 792 } 793 794 tabletFilters := getTabletFilters(filter) 795 796 rows := [][]sqltypes.Value{} 797 status := e.scatterConn.GetHealthCheckCacheStatus() 798 for _, s := range status { 799 for _, ts := range s.TabletsStats { 800 state := "SERVING" 801 if !ts.Serving { 802 state = "NOT_SERVING" 803 } 804 ptst := ts.PrimaryTermStartTime 805 ptstStr := "" 806 if ptst > 0 { 807 // this code depends on the fact that PrimaryTermStartTime is the seconds since epoch start 808 ptstStr = time.Unix(ptst, 0).UTC().Format(time.RFC3339) 809 } 810 811 skipTablet := false 812 for _, filter := range tabletFilters { 813 if !filter(ts.Tablet, state, ptst) { 814 skipTablet = true 815 break 816 } 817 } 818 819 if skipTablet { 820 continue 821 } 822 823 rows = append(rows, buildVarCharRow( 824 s.Cell, 825 s.Target.Keyspace, 826 s.Target.Shard, 827 ts.Target.TabletType.String(), 828 state, 829 topoproto.TabletAliasString(ts.Tablet.Alias), 830 ts.Tablet.Hostname, 831 ptstStr, 832 )) 833 } 834 } 835 return &sqltypes.Result{ 836 Fields: buildVarCharFields("Cell", "Keyspace", "Shard", "TabletType", "State", "Alias", "Hostname", "PrimaryTermStartTime"), 837 Rows: rows, 838 }, nil 839 } 840 841 func (e *Executor) showVitessReplicationStatus(ctx context.Context, filter *sqlparser.ShowFilter) (*sqltypes.Result, error) { 842 ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout) 843 defer cancel() 844 rows := [][]sqltypes.Value{} 845 846 status := e.scatterConn.GetHealthCheckCacheStatus() 847 848 for _, s := range status { 849 for _, ts := range s.TabletsStats { 850 // We only want to show REPLICA and RDONLY tablets 851 if ts.Tablet.Type != topodatapb.TabletType_REPLICA && ts.Tablet.Type != topodatapb.TabletType_RDONLY { 852 continue 853 } 854 855 // Allow people to filter by Keyspace and Shard using a LIKE clause 856 if filter != nil { 857 ksFilterRegex := sqlparser.LikeToRegexp(filter.Like) 858 keyspaceShardStr := fmt.Sprintf("%s/%s", ts.Tablet.Keyspace, ts.Tablet.Shard) 859 if !ksFilterRegex.MatchString(keyspaceShardStr) { 860 continue 861 } 862 } 863 864 tabletHostPort := ts.GetTabletHostPort() 865 throttlerStatus, err := getTabletThrottlerStatus(tabletHostPort) 866 if err != nil { 867 log.Warningf("Could not get throttler status from %s: %v", tabletHostPort, err) 868 } 869 870 replSourceHost := "" 871 replSourcePort := int64(0) 872 replIOThreadHealth := "" 873 replSQLThreadHealth := "" 874 replLastError := "" 875 replLag := int64(-1) 876 sql := "show slave status" 877 results, err := e.txConn.tabletGateway.Execute(ctx, ts.Target, sql, nil, 0, 0, nil) 878 if err != nil || results == nil { 879 log.Warningf("Could not get replication status from %s: %v", tabletHostPort, err) 880 } else if row := results.Named().Row(); row != nil { 881 replSourceHost = row["Master_Host"].ToString() 882 replSourcePort, _ = row["Master_Port"].ToInt64() 883 replIOThreadHealth = row["Slave_IO_Running"].ToString() 884 replSQLThreadHealth = row["Slave_SQL_Running"].ToString() 885 replLastError = row["Last_Error"].ToString() 886 if ts.Stats != nil { 887 replLag = int64(ts.Stats.ReplicationLagSeconds) 888 } 889 } 890 replicationHealth := fmt.Sprintf("{\"EventStreamRunning\":\"%s\",\"EventApplierRunning\":\"%s\",\"LastError\":\"%s\"}", replIOThreadHealth, replSQLThreadHealth, replLastError) 891 892 rows = append(rows, buildVarCharRow( 893 s.Target.Keyspace, 894 s.Target.Shard, 895 ts.Target.TabletType.String(), 896 topoproto.TabletAliasString(ts.Tablet.Alias), 897 ts.Tablet.Hostname, 898 fmt.Sprintf("%s:%d", replSourceHost, replSourcePort), 899 replicationHealth, 900 fmt.Sprintf("%d", replLag), 901 throttlerStatus, 902 )) 903 } 904 } 905 return &sqltypes.Result{ 906 Fields: buildVarCharFields("Keyspace", "Shard", "TabletType", "Alias", "Hostname", "ReplicationSource", "ReplicationHealth", "ReplicationLag", "ThrottlerStatus"), 907 Rows: rows, 908 }, nil 909 } 910 911 // MessageStream is part of the vtgate service API. This is a V2 level API that's sent 912 // to the Resolver. 913 func (e *Executor) MessageStream(ctx context.Context, keyspace string, shard string, keyRange *topodatapb.KeyRange, name string, callback func(*sqltypes.Result) error) error { 914 err := e.resolver.MessageStream( 915 ctx, 916 keyspace, 917 shard, 918 keyRange, 919 name, 920 callback, 921 ) 922 return formatError(err) 923 } 924 925 // VSchema returns the VSchema. 926 func (e *Executor) VSchema() *vindexes.VSchema { 927 e.mu.Lock() 928 defer e.mu.Unlock() 929 return e.vschema 930 } 931 932 // SaveVSchema updates the vschema and stats 933 func (e *Executor) SaveVSchema(vschema *vindexes.VSchema, stats *VSchemaStats) { 934 e.mu.Lock() 935 defer e.mu.Unlock() 936 if vschema != nil { 937 e.vschema = vschema 938 } 939 e.vschemaStats = stats 940 e.plans.Clear() 941 942 if vschemaCounters != nil { 943 vschemaCounters.Add("Reload", 1) 944 } 945 946 } 947 948 // ParseDestinationTarget parses destination target string and sets default keyspace if possible. 949 func (e *Executor) ParseDestinationTarget(targetString string) (string, topodatapb.TabletType, key.Destination, error) { 950 destKeyspace, destTabletType, dest, err := topoproto.ParseDestination(targetString, defaultTabletType) 951 // Set default keyspace 952 if destKeyspace == "" && len(e.VSchema().Keyspaces) == 1 { 953 for k := range e.VSchema().Keyspaces { 954 destKeyspace = k 955 } 956 } 957 return destKeyspace, destTabletType, dest, err 958 } 959 960 type iQueryOption interface { 961 cachePlan() bool 962 getSelectLimit() int 963 } 964 965 // getPlan computes the plan for the given query. If one is in 966 // the cache, it reuses it. 967 func (e *Executor) getPlan(ctx context.Context, vcursor *vcursorImpl, sql string, comments sqlparser.MarginComments, bindVars map[string]*querypb.BindVariable, qo iQueryOption, logStats *logstats.LogStats) (*engine.Plan, sqlparser.Statement, error) { 968 if e.VSchema() == nil { 969 return nil, nil, errors.New("vschema not initialized") 970 } 971 972 stmt, reserved, err := sqlparser.Parse2(sql) 973 if err != nil { 974 return nil, nil, err 975 } 976 query := sql 977 statement := stmt 978 reservedVars := sqlparser.NewReservedVars("vtg", reserved) 979 bindVarNeeds := &sqlparser.BindVarNeeds{} 980 if !sqlparser.IgnoreMaxPayloadSizeDirective(statement) && !isValidPayloadSize(query) { 981 return nil, nil, vterrors.NewErrorf(vtrpcpb.Code_RESOURCE_EXHAUSTED, vterrors.NetPacketTooLarge, "query payload size above threshold") 982 } 983 ignoreMaxMemoryRows := sqlparser.IgnoreMaxMaxMemoryRowsDirective(stmt) 984 vcursor.SetIgnoreMaxMemoryRows(ignoreMaxMemoryRows) 985 consolidator := sqlparser.Consolidator(stmt) 986 vcursor.SetConsolidator(consolidator) 987 988 setVarComment, err := prepareSetVarComment(vcursor, stmt) 989 if err != nil { 990 return nil, nil, err 991 } 992 // Normalize if possible and retry. 993 if e.canNormalizeStatement(stmt, qo, setVarComment) { 994 parameterize := e.normalize // the public flag is called normalize 995 result, err := sqlparser.PrepareAST( 996 stmt, 997 reservedVars, 998 bindVars, 999 parameterize, 1000 vcursor.keyspace, 1001 qo.getSelectLimit(), 1002 setVarComment, 1003 vcursor.safeSession.SystemVariables, 1004 vcursor, 1005 ) 1006 if err != nil { 1007 return nil, nil, err 1008 } 1009 statement = result.AST 1010 bindVarNeeds = result.BindVarNeeds 1011 query = sqlparser.String(statement) 1012 } 1013 1014 logStats.SQL = comments.Leading + query + comments.Trailing 1015 logStats.BindVariables = sqltypes.CopyBindVariables(bindVars) 1016 1017 return e.cacheAndBuildStatement(ctx, vcursor, query, statement, qo, logStats, stmt, reservedVars, bindVarNeeds) 1018 } 1019 1020 func (e *Executor) cacheAndBuildStatement(ctx context.Context, vcursor *vcursorImpl, query string, statement sqlparser.Statement, qo iQueryOption, logStats *logstats.LogStats, stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, bindVarNeeds *sqlparser.BindVarNeeds) (*engine.Plan, sqlparser.Statement, error) { 1021 planHash := sha256.New() 1022 _, _ = planHash.Write([]byte(vcursor.planPrefixKey(ctx))) 1023 _, _ = planHash.Write([]byte{':'}) 1024 _, _ = planHash.Write(hack.StringBytes(query)) 1025 planKey := hex.EncodeToString(planHash.Sum(nil)) 1026 1027 if sqlparser.CachePlan(statement) && qo.cachePlan() { 1028 if plan, ok := e.plans.Get(planKey); ok { 1029 logStats.CachedPlan = true 1030 return plan.(*engine.Plan), stmt, nil 1031 } 1032 } 1033 1034 plan, err := planbuilder.BuildFromStmt(query, statement, reservedVars, vcursor, bindVarNeeds, enableOnlineDDL, enableDirectDDL) 1035 if err != nil { 1036 return nil, nil, err 1037 } 1038 1039 plan.Warnings = vcursor.warnings 1040 vcursor.warnings = nil 1041 1042 err = e.checkThatPlanIsValid(stmt, plan) 1043 // Only cache the plan if it is valid (i.e. does not scatter) 1044 if err == nil && qo.cachePlan() && sqlparser.CachePlan(statement) { 1045 e.plans.Set(planKey, plan) 1046 } 1047 return plan, stmt, err 1048 } 1049 1050 func (e *Executor) canNormalizeStatement(stmt sqlparser.Statement, qo iQueryOption, setVarComment string) bool { 1051 return (e.normalize && sqlparser.CanNormalize(stmt)) || 1052 sqlparser.MustRewriteAST(stmt, qo.getSelectLimit() > 0) || setVarComment != "" 1053 } 1054 1055 func prepareSetVarComment(vcursor *vcursorImpl, stmt sqlparser.Statement) (string, error) { 1056 if vcursor == nil || vcursor.Session().InReservedConn() { 1057 return "", nil 1058 } 1059 1060 if !vcursor.Session().HasSystemVariables() { 1061 return "", nil 1062 } 1063 1064 switch stmt.(type) { 1065 // If the statement is a transaction statement or a set no reserved connection / SET_VAR is needed 1066 case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback, *sqlparser.Savepoint, 1067 *sqlparser.SRollback, *sqlparser.Release, *sqlparser.Set, *sqlparser.Show: 1068 return "", nil 1069 case sqlparser.SupportOptimizerHint: 1070 break 1071 default: 1072 vcursor.NeedsReservedConn() 1073 return "", nil 1074 } 1075 1076 var res strings.Builder 1077 vcursor.Session().GetSystemVariables(func(k, v string) { 1078 res.WriteString(fmt.Sprintf("SET_VAR(%s = %s) ", k, v)) 1079 }) 1080 return strings.TrimSpace(res.String()), nil 1081 } 1082 1083 func (e *Executor) debugGetPlan(planKey string) (*engine.Plan, bool) { 1084 planHash := sha256.Sum256([]byte(planKey)) 1085 planHex := hex.EncodeToString(planHash[:]) 1086 if plan, ok := e.plans.Get(planHex); ok { 1087 return plan.(*engine.Plan), true 1088 } 1089 return nil, false 1090 } 1091 1092 type cacheItem struct { 1093 Key string 1094 Value *engine.Plan 1095 } 1096 1097 func (e *Executor) debugCacheEntries() (items []cacheItem) { 1098 e.plans.ForEach(func(value any) bool { 1099 plan := value.(*engine.Plan) 1100 items = append(items, cacheItem{ 1101 Key: plan.Original, 1102 Value: plan, 1103 }) 1104 return true 1105 }) 1106 return 1107 } 1108 1109 // ServeHTTP shows the current plans in the query cache. 1110 func (e *Executor) ServeHTTP(response http.ResponseWriter, request *http.Request) { 1111 if err := acl.CheckAccessHTTP(request, acl.DEBUGGING); err != nil { 1112 acl.SendError(response, err) 1113 return 1114 } 1115 1116 switch request.URL.Path { 1117 case pathQueryPlans: 1118 returnAsJSON(response, e.debugCacheEntries()) 1119 case pathVSchema: 1120 returnAsJSON(response, e.VSchema()) 1121 case pathScatterStats: 1122 e.WriteScatterStats(response) 1123 default: 1124 response.WriteHeader(http.StatusNotFound) 1125 } 1126 } 1127 1128 func returnAsJSON(response http.ResponseWriter, stuff any) { 1129 response.Header().Set("Content-Type", "application/json; charset=utf-8") 1130 buf, err := json.MarshalIndent(stuff, "", " ") 1131 if err != nil { 1132 _, _ = response.Write([]byte(err.Error())) 1133 return 1134 } 1135 ebuf := bytes.NewBuffer(nil) 1136 json.HTMLEscape(ebuf, buf) 1137 _, _ = response.Write(ebuf.Bytes()) 1138 } 1139 1140 // Plans returns the LRU plan cache 1141 func (e *Executor) Plans() cache.Cache { 1142 return e.plans 1143 } 1144 1145 func (e *Executor) updateQueryCounts(planType, keyspace, tableName string, shardQueries int64) { 1146 queriesProcessed.Add(planType, 1) 1147 queriesRouted.Add(planType, shardQueries) 1148 if tableName != "" { 1149 queriesProcessedByTable.Add([]string{planType, keyspace, tableName}, 1) 1150 queriesRoutedByTable.Add([]string{planType, keyspace, tableName}, shardQueries) 1151 } 1152 } 1153 1154 // VSchemaStats returns the loaded vschema stats. 1155 func (e *Executor) VSchemaStats() *VSchemaStats { 1156 e.mu.Lock() 1157 defer e.mu.Unlock() 1158 if e.vschemaStats == nil { 1159 return &VSchemaStats{ 1160 Error: "No VSchema loaded yet.", 1161 } 1162 } 1163 return e.vschemaStats 1164 } 1165 1166 func buildVarCharFields(names ...string) []*querypb.Field { 1167 fields := make([]*querypb.Field, len(names)) 1168 for i, v := range names { 1169 fields[i] = &querypb.Field{ 1170 Name: v, 1171 Type: sqltypes.VarChar, 1172 Charset: collations.CollationUtf8ID, 1173 Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), 1174 } 1175 } 1176 return fields 1177 } 1178 1179 func buildVarCharRow(values ...string) []sqltypes.Value { 1180 row := make([]sqltypes.Value, len(values)) 1181 for i, v := range values { 1182 row[i] = sqltypes.NewVarChar(v) 1183 } 1184 return row 1185 } 1186 1187 // isValidPayloadSize validates whether a query payload is above the 1188 // configured MaxPayloadSize threshold. The WarnPayloadSizeExceeded will increment 1189 // if the payload size exceeds the warnPayloadSize. 1190 1191 func isValidPayloadSize(query string) bool { 1192 payloadSize := len(query) 1193 if maxPayloadSize > 0 && payloadSize > maxPayloadSize { 1194 return false 1195 } 1196 if warnPayloadSize > 0 && payloadSize > warnPayloadSize { 1197 warnings.Add("WarnPayloadSizeExceeded", 1) 1198 } 1199 return true 1200 } 1201 1202 // Prepare executes a prepare statements. 1203 func (e *Executor) Prepare(ctx context.Context, method string, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable) (fld []*querypb.Field, err error) { 1204 logStats := logstats.NewLogStats(ctx, method, sql, safeSession.GetSessionUUID(), bindVars) 1205 fld, err = e.prepare(ctx, safeSession, sql, bindVars, logStats) 1206 logStats.Error = err 1207 1208 // The mysql plugin runs an implicit rollback whenever a connection closes. 1209 // To avoid spamming the log with no-op rollback records, ignore it if 1210 // it was a no-op record (i.e. didn't issue any queries) 1211 if !(logStats.StmtType == "ROLLBACK" && logStats.ShardQueries == 0) { 1212 logStats.SaveEndTime() 1213 QueryLogger.Send(logStats) 1214 } 1215 return fld, err 1216 } 1217 1218 func (e *Executor) prepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) ([]*querypb.Field, error) { 1219 // Start an implicit transaction if necessary. 1220 if !safeSession.Autocommit && !safeSession.InTransaction() { 1221 if err := e.txConn.Begin(ctx, safeSession, nil); err != nil { 1222 return nil, err 1223 } 1224 } 1225 1226 if bindVars == nil { 1227 bindVars = make(map[string]*querypb.BindVariable) 1228 } 1229 1230 stmtType := sqlparser.Preview(sql) 1231 logStats.StmtType = stmtType.String() 1232 1233 // Mysql warnings are scoped to the current session, but are 1234 // cleared when a "non-diagnostic statement" is executed: 1235 // https://dev.mysql.com/doc/refman/8.0/en/show-warnings.html 1236 // 1237 // To emulate this behavior, clear warnings from the session 1238 // for all statements _except_ SHOW, so that SHOW WARNINGS 1239 // can actually return them. 1240 if stmtType != sqlparser.StmtShow { 1241 safeSession.ClearWarnings() 1242 } 1243 1244 switch stmtType { 1245 case sqlparser.StmtSelect, sqlparser.StmtShow: 1246 return e.handlePrepare(ctx, safeSession, sql, bindVars, logStats) 1247 case sqlparser.StmtDDL, sqlparser.StmtBegin, sqlparser.StmtCommit, sqlparser.StmtRollback, sqlparser.StmtSet, sqlparser.StmtInsert, sqlparser.StmtReplace, sqlparser.StmtUpdate, sqlparser.StmtDelete, 1248 sqlparser.StmtUse, sqlparser.StmtOther, sqlparser.StmtComment, sqlparser.StmtExplain, sqlparser.StmtFlush: 1249 return nil, nil 1250 } 1251 return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unrecognized prepare statement: %s", sql) 1252 } 1253 1254 func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) ([]*querypb.Field, error) { 1255 // V3 mode. 1256 query, comments := sqlparser.SplitMarginComments(sql) 1257 vcursor, _ := newVCursorImpl(safeSession, comments, e, logStats, e.vm, e.VSchema(), e.resolver.resolver, e.serv, e.warnShardedOnly, e.pv) 1258 plan, _, err := e.getPlan(ctx, vcursor, query, comments, bindVars, safeSession, logStats) 1259 execStart := time.Now() 1260 logStats.PlanTime = execStart.Sub(logStats.StartTime) 1261 1262 if err != nil { 1263 logStats.Error = err 1264 return nil, err 1265 } 1266 1267 err = e.addNeededBindVars(plan.BindVarNeeds, bindVars, safeSession) 1268 if err != nil { 1269 logStats.Error = err 1270 return nil, err 1271 } 1272 1273 qr, err := plan.Instructions.GetFields(ctx, vcursor, bindVars) 1274 logStats.ExecuteTime = time.Since(execStart) 1275 var errCount uint64 1276 if err != nil { 1277 logStats.Error = err 1278 errCount = 1 // nolint 1279 return nil, err 1280 } 1281 logStats.RowsAffected = qr.RowsAffected 1282 1283 plan.AddStats(1, time.Since(logStats.StartTime), logStats.ShardQueries, qr.RowsAffected, uint64(len(qr.Rows)), errCount) 1284 1285 return qr.Fields, err 1286 } 1287 1288 // ExecuteMultiShard implements the IExecutor interface 1289 func (e *Executor) ExecuteMultiShard(ctx context.Context, primitive engine.Primitive, rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, session *SafeSession, autocommit bool, ignoreMaxMemoryRows bool) (qr *sqltypes.Result, errs []error) { 1290 return e.scatterConn.ExecuteMultiShard(ctx, primitive, rss, queries, session, autocommit, ignoreMaxMemoryRows) 1291 } 1292 1293 // StreamExecuteMulti implements the IExecutor interface 1294 func (e *Executor) StreamExecuteMulti(ctx context.Context, primitive engine.Primitive, query string, rss []*srvtopo.ResolvedShard, vars []map[string]*querypb.BindVariable, session *SafeSession, autocommit bool, callback func(reply *sqltypes.Result) error) []error { 1295 return e.scatterConn.StreamExecuteMulti(ctx, primitive, query, rss, vars, session, autocommit, callback) 1296 } 1297 1298 // ExecuteLock implements the IExecutor interface 1299 func (e *Executor) ExecuteLock(ctx context.Context, rs *srvtopo.ResolvedShard, query *querypb.BoundQuery, session *SafeSession, lockFuncType sqlparser.LockingFuncType) (*sqltypes.Result, error) { 1300 return e.scatterConn.ExecuteLock(ctx, rs, query, session, lockFuncType) 1301 } 1302 1303 // ExecuteMessageStream implements the IExecutor interface 1304 func (e *Executor) ExecuteMessageStream(ctx context.Context, rss []*srvtopo.ResolvedShard, tableName string, callback func(reply *sqltypes.Result) error) error { 1305 return e.scatterConn.MessageStream(ctx, rss, tableName, callback) 1306 } 1307 1308 // ExecuteVStream implements the IExecutor interface 1309 func (e *Executor) ExecuteVStream(ctx context.Context, rss []*srvtopo.ResolvedShard, filter *binlogdatapb.Filter, gtid string, callback func(evs []*binlogdatapb.VEvent) error) error { 1310 return e.startVStream(ctx, rss, filter, gtid, callback) 1311 } 1312 1313 func (e *Executor) startVStream(ctx context.Context, rss []*srvtopo.ResolvedShard, filter *binlogdatapb.Filter, gtid string, callback func(evs []*binlogdatapb.VEvent) error) error { 1314 var shardGtids []*binlogdatapb.ShardGtid 1315 for _, rs := range rss { 1316 shardGtid := &binlogdatapb.ShardGtid{ 1317 Keyspace: rs.Target.Keyspace, 1318 Shard: rs.Target.Shard, 1319 Gtid: gtid, 1320 } 1321 shardGtids = append(shardGtids, shardGtid) 1322 } 1323 vgtid := &binlogdatapb.VGtid{ 1324 ShardGtids: shardGtids, 1325 } 1326 ts, err := e.serv.GetTopoServer() 1327 if err != nil { 1328 return err 1329 } 1330 1331 vsm := newVStreamManager(e.resolver.resolver, e.serv, e.cell) 1332 vs := &vstream{ 1333 vgtid: vgtid, 1334 tabletType: topodatapb.TabletType_PRIMARY, 1335 filter: filter, 1336 send: callback, 1337 resolver: e.resolver.resolver, 1338 journaler: make(map[int64]*journalEvent), 1339 skewTimeoutSeconds: maxSkewTimeoutSeconds, 1340 timestamps: make(map[string]int64), 1341 vsm: vsm, 1342 eventCh: make(chan []*binlogdatapb.VEvent), 1343 ts: ts, 1344 copyCompletedShard: make(map[string]struct{}), 1345 } 1346 _ = vs.stream(ctx) 1347 return nil 1348 } 1349 1350 func (e *Executor) checkThatPlanIsValid(stmt sqlparser.Statement, plan *engine.Plan) error { 1351 if e.allowScatter || plan.Instructions == nil || sqlparser.AllowScatterDirective(stmt) { 1352 return nil 1353 } 1354 // we go over all the primitives in the plan, searching for a route that is of SelectScatter opcode 1355 badPrimitive := engine.Find(func(node engine.Primitive) bool { 1356 router, ok := node.(*engine.Route) 1357 if !ok { 1358 return false 1359 } 1360 return router.Opcode == engine.Scatter 1361 }, plan.Instructions) 1362 1363 if badPrimitive == nil { 1364 return nil 1365 } 1366 1367 return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "plan includes scatter, which is disallowed using the `no_scatter` command line argument") 1368 } 1369 1370 func getTabletThrottlerStatus(tabletHostPort string) (string, error) { 1371 client := http.Client{ 1372 Timeout: 100 * time.Millisecond, 1373 } 1374 resp, err := client.Get(fmt.Sprintf("http://%s/throttler/check?app=vtgate", tabletHostPort)) 1375 if err != nil { 1376 return "", err 1377 } 1378 defer resp.Body.Close() 1379 body, err := io.ReadAll(resp.Body) 1380 if err != nil { 1381 return "", err 1382 } 1383 1384 var elements struct { 1385 StatusCode int 1386 Value float64 1387 Threshold float64 1388 Message string 1389 } 1390 err = json.Unmarshal(body, &elements) 1391 if err != nil { 1392 return "", err 1393 } 1394 1395 httpStatusStr := http.StatusText(elements.StatusCode) 1396 1397 load := float64(0) 1398 if elements.Threshold > 0 { 1399 load = float64((elements.Value / elements.Threshold) * 100) 1400 } 1401 1402 status := fmt.Sprintf("{\"state\":\"%s\",\"load\":%.2f,\"message\":\"%s\"}", httpStatusStr, load, elements.Message) 1403 return status, nil 1404 } 1405 1406 // ReleaseLock implements the IExecutor interface 1407 func (e *Executor) ReleaseLock(ctx context.Context, session *SafeSession) error { 1408 return e.txConn.ReleaseLock(ctx, session) 1409 }