github.com/whtcorpsinc/MilevaDB-Prod@v0.0.0-20211104133533-f57f4be3b597/causetstore/stochastik/session.go

// Copyright 2020 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.

// Copyright 2020 WHTCORPS INC, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package stochastik

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net"
	"runtime/trace"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/ngaut/pools"
	"github.com/opentracing/opentracing-go"
	"github.com/whtcorpsinc/BerolinaSQL"
	"github.com/whtcorpsinc/BerolinaSQL/allegrosql"
	"github.com/whtcorpsinc/BerolinaSQL/ast"
	"github.com/whtcorpsinc/BerolinaSQL/auth"
	"github.com/whtcorpsinc/BerolinaSQL/charset"
	"github.com/whtcorpsinc/BerolinaSQL/perceptron"
	"github.com/whtcorpsinc/BerolinaSQL/terror"
	"github.com/whtcorpsinc/errors"
	"github.com/whtcorpsinc/failpoint"
	"github.com/whtcorpsinc/fidelpb/go-binlog"
	"github.com/whtcorpsinc/milevadb/bindinfo"
	"github.com/whtcorpsinc/milevadb/causet"
	causetembedded "github.com/whtcorpsinc/milevadb/causet/embedded"
	"github.com/whtcorpsinc/milevadb/causetstore/einsteindb"
	"github.com/whtcorpsinc/milevadb/config"
	"github.com/whtcorpsinc/milevadb/ekv"
	"github.com/whtcorpsinc/milevadb/interlock"
	"github.com/whtcorpsinc/milevadb/metrics"
	"github.com/whtcorpsinc/milevadb/petri"
	"github.com/whtcorpsinc/milevadb/plugin"
	"github.com/whtcorpsinc/milevadb/privilege"
	"github.com/whtcorpsinc/milevadb/privilege/privileges"
	"github.com/whtcorpsinc/milevadb/schemareplicant"
	"github.com/whtcorpsinc/milevadb/soliton"
	"github.com/whtcorpsinc/milevadb/soliton/chunk"
	"github.com/whtcorpsinc/milevadb/soliton/collate"
	"github.com/whtcorpsinc/milevadb/soliton/ekvcache"
	"github.com/whtcorpsinc/milevadb/soliton/execdetails"
	"github.com/whtcorpsinc/milevadb/soliton/logutil"
	"github.com/whtcorpsinc/milevadb/soliton/sqlexec"
	"github.com/whtcorpsinc/milevadb/soliton/timeutil"
	"github.com/whtcorpsinc/milevadb/spacetime"
	"github.com/whtcorpsinc/milevadb/statistics/handle"
	"github.com/whtcorpsinc/milevadb/stochastikctx"
	"github.com/whtcorpsinc/milevadb/stochastikctx/binloginfo"
	"github.com/whtcorpsinc/milevadb/stochastikctx/stmtctx"
	"github.com/whtcorpsinc/milevadb/stochastikctx/variable"
	"github.com/whtcorpsinc/milevadb/tenant"
	"github.com/whtcorpsinc/milevadb/types"
	"go.uber.org/zap"
)

var (
	memexPerTransactionPessimisticOK     = metrics.StatementPerTransaction.WithLabelValues(metrics.LblPessimistic, metrics.LblOK)
	memexPerTransactionPessimisticError  = metrics.StatementPerTransaction.WithLabelValues(metrics.LblPessimistic, metrics.LblError)
	memexPerTransactionOptimisticOK      = metrics.StatementPerTransaction.WithLabelValues(metrics.LblOptimistic, metrics.LblOK)
	memexPerTransactionOptimisticError   = metrics.StatementPerTransaction.WithLabelValues(metrics.LblOptimistic, metrics.LblError)
	transactionDurationPessimisticCommit = metrics.TransactionDuration.WithLabelValues(metrics.LblPessimistic, metrics.LblCommit)
	transactionDurationPessimisticAbort  = metrics.TransactionDuration.WithLabelValues(metrics.LblPessimistic, metrics.LblAbort)
	transactionDurationOptimisticCommit  = metrics.TransactionDuration.WithLabelValues(metrics.LblOptimistic, metrics.LblCommit)
	transactionDurationOptimisticAbort   = metrics.TransactionDuration.WithLabelValues(metrics.LblOptimistic, metrics.LblAbort)

	stochastikInterDircuteCompileDurationInternal = metrics.StochastikInterDircuteCompileDuration.WithLabelValues(metrics.LblInternal)
	stochastikInterDircuteCompileDurationGeneral  = metrics.StochastikInterDircuteCompileDuration.WithLabelValues(metrics.LblGeneral)
	stochastikInterDircuteParseDurationInternal   = metrics.StochastikInterDircuteParseDuration.WithLabelValues(metrics.LblInternal)
	stochastikInterDircuteParseDurationGeneral    = metrics.StochastikInterDircuteParseDuration.WithLabelValues(metrics.LblGeneral)
)

// Stochastik context, it is consistent with the lifecycle of a client connection.
type Stochastik interface {
	stochastikctx.Context
	Status() uint16       // Flag of current status, such as autocommit.
	LastInsertID() uint64 // LastInsertID is the last inserted auto_increment ID.
	LastMessage() string  // LastMessage is the info message that may be generated by last command
	AffectedRows() uint64 // Affected rows by latest executed stmt.
	// InterDircute is deprecated, use InterDircuteStmt() instead.
	InterDircute(context.Context, string) ([]sqlexec.RecordSet, error)         // InterDircute a allegrosql memex.
	InterDircuteInternal(context.Context, string) ([]sqlexec.RecordSet, error) // InterDircute a internal allegrosql memex.
	InterDircuteStmt(context.Context, ast.StmtNode) (sqlexec.RecordSet, error)
	Parse(ctx context.Context, allegrosql string) ([]ast.StmtNode, error)
	String() string // String is used to debug.
	CommitTxn(context.Context) error
	RollbackTxn(context.Context)
	// PrepareStmt executes prepare memex in binary protocol.
	PrepareStmt(allegrosql string) (stmtID uint32, paramCount int, fields []*ast.ResultField, err error)
	// InterDircutePreparedStmt executes a prepared memex.
	InterDircutePreparedStmt(ctx context.Context, stmtID uint32, param []types.Causet) (sqlexec.RecordSet, error)
	DropPreparedStmt(stmtID uint32) error
	SetClientCapability(uint32) // Set client capability flags.
	SetConnectionID(uint64)
	SetCommandValue(byte)
	SetProcessInfo(string, time.Time, byte, uint64)
	SetTLSState(*tls.ConnectionState)
	SetDefCauslation(coID int) error
	SetStochastikManager(soliton.StochastikManager)
	Close()
	Auth(user *auth.UserIdentity, auth []byte, salt []byte) bool
	AuthWithoutVerification(user *auth.UserIdentity) bool
	ShowProcess() *soliton.ProcessInfo
	// PrepareTxnCtx is exported for test.
	PrepareTxnCtx(context.Context)
	// FieldList returns fields list of a causet.
	FieldList(blockName string) (fields []*ast.ResultField, err error)
}

var (
	_ Stochastik = (*stochastik)(nil)
)

type stmtRecord struct {
	st      sqlexec.Statement
	stmtCtx *stmtctx.StatementContext
}

// StmtHistory holds all histories of memexs in a txn.
type StmtHistory struct {
	history []*stmtRecord
}

// Add appends a stmt to history list.
func (h *StmtHistory) Add(st sqlexec.Statement, stmtCtx *stmtctx.StatementContext) {
	s := &stmtRecord{
		st:      st,
		stmtCtx: stmtCtx,
	}
	h.history = append(h.history, s)
}

// Count returns the count of the history.
func (h *StmtHistory) Count() int {
	return len(h.history)
}

type stochastik struct {
	// processInfo is used by ShowProcess(), and should be modified atomically.
	processInfo atomic.Value
	txn         TxnState

	mu struct {
		sync.RWMutex
		values map[fmt.Stringer]interface{}
	}

	currentCtx    context.Context // only use for runtime.trace, Please NEVER use it.
	currentCauset causetembedded.Causet

	causetstore ekv.CausetStorage

	BerolinaSQL *BerolinaSQL.BerolinaSQL

	preparedCausetCache *ekvcache.SimpleLRUCache

	stochastikVars    *variable.StochastikVars
	stochastikManager soliton.StochastikManager

	statsDefCauslector *handle.StochastikStatsDefCauslector
	// dbsTenantChecker is used in `select milevadb_is_dbs_tenant()` memex;
	dbsTenantChecker tenant.DBSTenantChecker
	// lockedTables use to record the causet locks hold by the stochastik.
	lockedTables map[int64]perceptron.TableLockTpInfo

	// client shared interlock client per stochastik
	client ekv.Client
}

// AddTableLock adds causet dagger to the stochastik dagger map.
func (s *stochastik) AddTableLock(locks []perceptron.TableLockTpInfo) {
	for _, l := range locks {
		s.lockedTables[l.TableID] = l
	}
}

// ReleaseTableLocks releases causet dagger in the stochastik dagger map.
func (s *stochastik) ReleaseTableLocks(locks []perceptron.TableLockTpInfo) {
	for _, l := range locks {
		delete(s.lockedTables, l.TableID)
	}
}

// ReleaseTableLockByTableIDs releases causet dagger in the stochastik dagger map by causet ID.
func (s *stochastik) ReleaseTableLockByTableIDs(blockIDs []int64) {
	for _, tblID := range blockIDs {
		delete(s.lockedTables, tblID)
	}
}

// CheckTableLocked checks the causet dagger.
func (s *stochastik) CheckTableLocked(tblID int64) (bool, perceptron.TableLockType) {
	lt, ok := s.lockedTables[tblID]
	if !ok {
		return false, perceptron.TableLockNone
	}
	return true, lt.Tp
}

// GetAllTableLocks gets all causet locks causet id and EDB id hold by the stochastik.
func (s *stochastik) GetAllTableLocks() []perceptron.TableLockTpInfo {
	lockTpInfo := make([]perceptron.TableLockTpInfo, 0, len(s.lockedTables))
	for _, tl := range s.lockedTables {
		lockTpInfo = append(lockTpInfo, tl)
	}
	return lockTpInfo
}

// HasLockedTables uses to check whether this stochastik locked any blocks.
// If so, the stochastik can only visit the causet which locked by self.
func (s *stochastik) HasLockedTables() bool {
	b := len(s.lockedTables) > 0
	return b
}

// ReleaseAllTableLocks releases all causet locks hold by the stochastik.
func (s *stochastik) ReleaseAllTableLocks() {
	s.lockedTables = make(map[int64]perceptron.TableLockTpInfo)
}

// DBSTenantChecker returns s.dbsTenantChecker.
func (s *stochastik) DBSTenantChecker() tenant.DBSTenantChecker {
	return s.dbsTenantChecker
}

func (s *stochastik) cleanRetryInfo() {
	if s.stochastikVars.RetryInfo.Retrying {
		return
	}

	retryInfo := s.stochastikVars.RetryInfo
	defer retryInfo.Clean()
	if len(retryInfo.DroppedPreparedStmtIDs) == 0 {
		return
	}

	planCacheEnabled := causetembedded.PreparedCausetCacheEnabled()
	var cacheKey ekvcache.Key
	var preparedAst *ast.Prepared
	if planCacheEnabled {
		firstStmtID := retryInfo.DroppedPreparedStmtIDs[0]
		if preparedPointer, ok := s.stochastikVars.PreparedStmts[firstStmtID]; ok {
			preparedObj, ok := preparedPointer.(*causetembedded.CachedPrepareStmt)
			if ok {
				preparedAst = preparedObj.PreparedAst
				cacheKey = causetembedded.NewPSTMTCausetCacheKey(s.stochastikVars, firstStmtID, preparedAst.SchemaVersion)
			}
		}
	}
	for i, stmtID := range retryInfo.DroppedPreparedStmtIDs {
		if planCacheEnabled {
			if i > 0 && preparedAst != nil {
				causetembedded.SetPstmtIDSchemaVersion(cacheKey, stmtID, preparedAst.SchemaVersion, s.stochastikVars.IsolationReadEngines)
			}
			s.PreparedCausetCache().Delete(cacheKey)
		}
		s.stochastikVars.RemovePreparedStmt(stmtID)
	}
}

func (s *stochastik) Status() uint16 {
	return s.stochastikVars.Status
}

func (s *stochastik) LastInsertID() uint64 {
	if s.stochastikVars.StmtCtx.LastInsertID > 0 {
		return s.stochastikVars.StmtCtx.LastInsertID
	}
	return s.stochastikVars.StmtCtx.InsertID
}

func (s *stochastik) LastMessage() string {
	return s.stochastikVars.StmtCtx.GetMessage()
}

func (s *stochastik) AffectedRows() uint64 {
	return s.stochastikVars.StmtCtx.AffectedRows()
}

func (s *stochastik) SetClientCapability(capability uint32) {
	s.stochastikVars.ClientCapability = capability
}

func (s *stochastik) SetConnectionID(connectionID uint64) {
	s.stochastikVars.ConnectionID = connectionID
}

func (s *stochastik) SetTLSState(tlsState *tls.ConnectionState) {
	// If user is not connected via TLS, then tlsState == nil.
	if tlsState != nil {
		s.stochastikVars.TLSConnectionState = tlsState
	}
}

func (s *stochastik) SetCommandValue(command byte) {
	atomic.StoreUint32(&s.stochastikVars.CommandValue, uint32(command))
}

func (s *stochastik) SetDefCauslation(coID int) error {
	cs, co, err := charset.GetCharsetInfoByID(coID)
	if err != nil {
		return err
	}
	for _, v := range variable.SetNamesVariables {
		terror.Log(s.stochastikVars.SetSystemVar(v, cs))
	}
	return s.stochastikVars.SetSystemVar(variable.DefCauslationConnection, co)
}

func (s *stochastik) PreparedCausetCache() *ekvcache.SimpleLRUCache {
	return s.preparedCausetCache
}

func (s *stochastik) SetStochastikManager(sm soliton.StochastikManager) {
	s.stochastikManager = sm
}

func (s *stochastik) GetStochastikManager() soliton.StochastikManager {
	return s.stochastikManager
}

func (s *stochastik) StoreQueryFeedback(feedback interface{}) {
	if s.statsDefCauslector != nil {
		do, err := GetPetri(s.causetstore)
		if err != nil {
			logutil.BgLogger().Debug("petri not found", zap.Error(err))
			metrics.StoreQueryFeedbackCounter.WithLabelValues(metrics.LblError).Inc()
			return
		}
		err = s.statsDefCauslector.StoreQueryFeedback(feedback, do.StatsHandle())
		if err != nil {
			logutil.BgLogger().Debug("causetstore query feedback", zap.Error(err))
			metrics.StoreQueryFeedbackCounter.WithLabelValues(metrics.LblError).Inc()
			return
		}
		metrics.StoreQueryFeedbackCounter.WithLabelValues(metrics.LblOK).Inc()
	}
}

// FieldList returns fields list of a causet.
func (s *stochastik) FieldList(blockName string) ([]*ast.ResultField, error) {
	is := schemareplicant.GetSchemaReplicant(s)
	dbName := perceptron.NewCIStr(s.GetStochastikVars().CurrentDB)
	tName := perceptron.NewCIStr(blockName)
	pm := privilege.GetPrivilegeManager(s)
	if pm != nil && s.stochastikVars.User != nil {
		if !pm.RequestVerification(s.stochastikVars.ActiveRoles, dbName.O, tName.O, "", allegrosql.AllPrivMask) {
			user := s.stochastikVars.User
			u := user.Username
			h := user.Hostname
			if len(user.AuthUsername) > 0 && len(user.AuthHostname) > 0 {
				u = user.AuthUsername
				h = user.AuthHostname
			}
			return nil, causetembedded.ErrTableaccessDenied.GenWithStackByArgs("SELECT", u, h, blockName)
		}
	}
	causet, err := is.TableByName(dbName, tName)
	if err != nil {
		return nil, err
	}

	defcaus := causet.DefCauss()
	fields := make([]*ast.ResultField, 0, len(defcaus))
	for _, col := range causet.DefCauss() {
		rf := &ast.ResultField{
			DeferredCausetAsName: col.Name,
			TableAsName:          tName,
			DBName:               dbName,
			Block:                causet.Meta(),
			DeferredCauset:       col.DeferredCausetInfo,
		}
		fields = append(fields, rf)
	}
	return fields, nil
}

func (s *stochastik) doCommit(ctx context.Context) error {
	if !s.txn.Valid() {
		return nil
	}
	defer func() {
		s.txn.changeToInvalid()
		s.stochastikVars.SetStatusFlag(allegrosql.ServerStatusInTrans, false)
	}()
	if s.txn.IsReadOnly() {
		return nil
	}

	// mockCommitError and mockGetTSErrorInRetry use to test PR #8743.
	failpoint.Inject("mockCommitError", func(val failpoint.Value) {
		if val.(bool) && ekv.IsMockCommitErrorEnable() {
			ekv.MockCommitErrorDisable()
			failpoint.Return(ekv.ErrTxnRetryable)
		}
	})

	if s.stochastikVars.BinlogClient != nil {
		prewriteValue := binloginfo.GetPrewriteValue(s, false)
		if prewriteValue != nil {
			prewriteData, err := prewriteValue.Marshal()
			if err != nil {
				return errors.Trace(err)
			}
			info := &binloginfo.BinlogInfo{
				Data: &binlog.Binlog{
					Tp:            binlog.BinlogType_Prewrite,
					PrewriteValue: prewriteData,
				},
				Client: s.stochastikVars.BinlogClient,
			}
			s.txn.SetOption(ekv.BinlogInfo, info)
		}
	}

	// Get the related causet or partition IDs.
	relatedPhysicalTables := s.GetStochastikVars().TxnCtx.TableDeltaMap
	physicalTableIDs := make([]int64, 0, len(relatedPhysicalTables))
	for id := range relatedPhysicalTables {
		physicalTableIDs = append(physicalTableIDs, id)
	}
	// Set this option for 2 phase commit to validate schemaReplicant lease.
	s.txn.SetOption(ekv.SchemaChecker, petri.NewSchemaChecker(petri.GetPetri(s), s.stochastikVars.TxnCtx.SchemaVersion, physicalTableIDs))
	s.txn.SetOption(ekv.SchemaReplicant, s.stochastikVars.TxnCtx.SchemaReplicant)
	s.txn.SetOption(ekv.CommitHook, func(info ekv.TxnInfo, _ error) { s.stochastikVars.LastTxnInfo = info })
	if s.GetStochastikVars().EnableAmendPessimisticTxn {
		s.txn.SetOption(ekv.SchemaAmender, NewSchemaAmenderForEinsteinDBTxn(s))
	}

	return s.txn.Commit(stochastikctx.SetCommitCtx(ctx, s))
}

func (s *stochastik) doCommitWithRetry(ctx context.Context) error {
	defer func() {
		s.GetStochastikVars().SetTxnIsolationLevelOneShotStateForNextTxn()
		s.txn.changeToInvalid()
		s.cleanRetryInfo()
	}()
	if !s.txn.Valid() {
		// If the transaction is invalid, maybe it has already been rolled back by the client.
		return nil
	}
	txnSize := s.txn.Size()
	isPessimistic := s.txn.IsPessimistic()
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.doCommitWitRetry", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
	}
	err := s.doCommit(ctx)
	if err != nil {
		commitRetryLimit := s.stochastikVars.RetryLimit
		if !s.stochastikVars.TxnCtx.CouldRetry {
			commitRetryLimit = 0
		}
		// Don't retry in BatchInsert mode. As a counter-example, insert into t1 select * from t2,
		// BatchInsert already commit the first batch 1000 rows, then it commit 1000-2000 and retry the memex,
		// Finally t1 will have more data than t2, with no errors return to user!
		if s.isTxnRetryableError(err) && !s.stochastikVars.BatchInsert && commitRetryLimit > 0 && !isPessimistic {
			logutil.Logger(ctx).Warn("allegrosql",
				zap.String("label", s.getALLEGROSQLLabel()),
				zap.Error(err),
				zap.String("txn", s.txn.GoString()))
			// Transactions will retry 2 ~ commitRetryLimit times.
			// We make larger transactions retry less times to prevent cluster resource outage.
			txnSizeRate := float64(txnSize) / float64(ekv.TxnTotalSizeLimit)
			maxRetryCount := commitRetryLimit - int64(float64(commitRetryLimit-1)*txnSizeRate)
			err = s.retry(ctx, uint(maxRetryCount))
		} else {
			logutil.Logger(ctx).Warn("can not retry txn",
				zap.String("label", s.getALLEGROSQLLabel()),
				zap.Error(err),
				zap.Bool("IsBatchInsert", s.stochastikVars.BatchInsert),
				zap.Bool("IsPessimistic", isPessimistic),
				zap.Bool("InRestrictedALLEGROSQL", s.stochastikVars.InRestrictedALLEGROSQL),
				zap.Int64("milevadb_retry_limit", s.stochastikVars.RetryLimit),
				zap.Bool("milevadb_disable_txn_auto_retry", s.stochastikVars.DisableTxnAutoRetry))
		}
	}
	counter := s.stochastikVars.TxnCtx.StatementCount
	duration := time.Since(s.GetStochastikVars().TxnCtx.CreateTime).Seconds()
	s.recordOnTransactionInterDircution(err, counter, duration)

	if err != nil {
		logutil.Logger(ctx).Warn("commit failed",
			zap.String("finished txn", s.txn.GoString()),
			zap.Error(err))
		return err
	}
	mapper := s.GetStochastikVars().TxnCtx.TableDeltaMap
	if s.statsDefCauslector != nil && mapper != nil {
		for id, item := range mapper {
			s.statsDefCauslector.UFIDelate(id, item.Delta, item.Count, &item.DefCausSize)
		}
	}
	return nil
}

func (s *stochastik) CommitTxn(ctx context.Context) error {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.CommitTxn", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
	}

	var commitDetail *execdetails.CommitDetails
	ctx = context.WithValue(ctx, execdetails.CommitDetailCtxKey, &commitDetail)
	err := s.doCommitWithRetry(ctx)
	if commitDetail != nil {
		s.stochastikVars.StmtCtx.MergeInterDircDetails(nil, commitDetail)
	}

	failpoint.Inject("keepHistory", func(val failpoint.Value) {
		if val.(bool) {
			failpoint.Return(err)
		}
	})

	s.stochastikVars.TxnCtx.Cleanup()
	return err
}

func (s *stochastik) RollbackTxn(ctx context.Context) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.RollbackTxn", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
	}

	if s.txn.Valid() {
		terror.Log(s.txn.Rollback())
	}
	if ctx.Value(inCloseStochastik{}) == nil {
		s.cleanRetryInfo()
	}
	s.txn.changeToInvalid()
	s.stochastikVars.TxnCtx.Cleanup()
	s.stochastikVars.SetStatusFlag(allegrosql.ServerStatusInTrans, false)
}

func (s *stochastik) GetClient() ekv.Client {
	return s.client
}

func (s *stochastik) String() string {
	// TODO: how to print binded context in values appropriately?
	sessVars := s.stochastikVars
	data := map[string]interface{}{
		"id":         sessVars.ConnectionID,
		"user":       sessVars.User,
		"currDBName": sessVars.CurrentDB,
		"status":     sessVars.Status,
		"strictMode": sessVars.StrictALLEGROSQLMode,
	}
	if s.txn.Valid() {
		// if txn is committed or rolled back, txn is nil.
		data["txn"] = s.txn.String()
	}
	if sessVars.SnapshotTS != 0 {
		data["snapshotTS"] = sessVars.SnapshotTS
	}
	if sessVars.StmtCtx.LastInsertID > 0 {
		data["lastInsertID"] = sessVars.StmtCtx.LastInsertID
	}
	if len(sessVars.PreparedStmts) > 0 {
		data["preparedStmtCount"] = len(sessVars.PreparedStmts)
	}
	b, err := json.MarshalIndent(data, "", "  ")
	terror.Log(errors.Trace(err))
	return string(b)
}

const sqlLogMaxLen = 1024

// SchemaChangedWithoutRetry is used for testing.
var SchemaChangedWithoutRetry uint32

func (s *stochastik) getALLEGROSQLLabel() string {
	if s.stochastikVars.InRestrictedALLEGROSQL {
		return metrics.LblInternal
	}
	return metrics.LblGeneral
}

func (s *stochastik) isInternal() bool {
	return s.stochastikVars.InRestrictedALLEGROSQL
}

func (s *stochastik) isTxnRetryableError(err error) bool {
	if atomic.LoadUint32(&SchemaChangedWithoutRetry) == 1 {
		return ekv.IsTxnRetryableError(err)
	}
	return ekv.IsTxnRetryableError(err) || petri.ErrSchemaReplicantChanged.Equal(err)
}

func (s *stochastik) checkTxnAborted(stmt sqlexec.Statement) error {
	var err error
	if atomic.LoadUint32(&s.GetStochastikVars().TxnCtx.LockExpire) > 0 {
		err = einsteindb.ErrLockExpire
	} else {
		return nil
	}
	// If the transaction is aborted, the following memexs do not need to execute, except `commit` and `rollback`,
	// because they are used to finish the aborted transaction.
	if _, ok := stmt.(*interlock.InterDircStmt).StmtNode.(*ast.CommitStmt); ok {
		return nil
	}
	if _, ok := stmt.(*interlock.InterDircStmt).StmtNode.(*ast.RollbackStmt); ok {
		return nil
	}
	return err
}

func (s *stochastik) retry(ctx context.Context, maxCnt uint) (err error) {
	var retryCnt uint
	defer func() {
		s.stochastikVars.RetryInfo.Retrying = false
		// retryCnt only increments on retryable error, so +1 here.
		metrics.StochastikRetry.Observe(float64(retryCnt + 1))
		s.stochastikVars.SetStatusFlag(allegrosql.ServerStatusInTrans, false)
		if err != nil {
			s.RollbackTxn(ctx)
		}
		s.txn.changeToInvalid()
	}()

	connID := s.stochastikVars.ConnectionID
	s.stochastikVars.RetryInfo.Retrying = true
	if atomic.LoadUint32(&s.stochastikVars.TxnCtx.ForUFIDelate) == 1 {
		err = ErrForUFIDelateCantRetry.GenWithStackByArgs(connID)
		return err
	}

	nh := GetHistory(s)
	var schemaVersion int64
	sessVars := s.GetStochastikVars()
	orgStartTS := sessVars.TxnCtx.StartTS
	label := s.getALLEGROSQLLabel()
	for {
		s.PrepareTxnCtx(ctx)
		s.stochastikVars.RetryInfo.ResetOffset()
		for i, sr := range nh.history {
			st := sr.st
			s.stochastikVars.StmtCtx = sr.stmtCtx
			s.stochastikVars.StmtCtx.ResetForRetry()
			s.stochastikVars.PreparedParams = s.stochastikVars.PreparedParams[:0]
			schemaVersion, err = st.RebuildCauset(ctx)
			if err != nil {
				return err
			}

			if retryCnt == 0 {
				// We do not have to log the query every time.
				// We print the queries at the first try only.
				allegrosql := sqlForLog(st.GetTextToLog())
				if !config.RedactLogEnabled() {
					allegrosql += sessVars.PreparedParams.String()
				}
				logutil.Logger(ctx).Warn("retrying",
					zap.Int64("schemaVersion", schemaVersion),
					zap.Uint("retryCnt", retryCnt),
					zap.Int("queryNum", i),
					zap.String("allegrosql", allegrosql))
			} else {
				logutil.Logger(ctx).Warn("retrying",
					zap.Int64("schemaVersion", schemaVersion),
					zap.Uint("retryCnt", retryCnt),
					zap.Int("queryNum", i))
			}
			_, err = st.InterDirc(ctx)
			if err != nil {
				s.StmtRollback()
				break
			}
			s.StmtCommit()
		}
		logutil.Logger(ctx).Warn("transaction association",
			zap.Uint64("retrying txnStartTS", s.GetStochastikVars().TxnCtx.StartTS),
			zap.Uint64("original txnStartTS", orgStartTS))
		failpoint.Inject("preCommitHook", func() {
			hook, ok := ctx.Value("__preCommitHook").(func())
			if ok {
				hook()
			}
		})
		if err == nil {
			err = s.doCommit(ctx)
			if err == nil {
				break
			}
		}
		if !s.isTxnRetryableError(err) {
			logutil.Logger(ctx).Warn("allegrosql",
				zap.String("label", label),
				zap.Stringer("stochastik", s),
				zap.Error(err))
			metrics.StochastikRetryErrorCounter.WithLabelValues(label, metrics.LblUnretryable).Inc()
			return err
		}
		retryCnt++
		if retryCnt >= maxCnt {
			logutil.Logger(ctx).Warn("allegrosql",
				zap.String("label", label),
				zap.Uint("retry reached max count", retryCnt))
			metrics.StochastikRetryErrorCounter.WithLabelValues(label, metrics.LblReachMax).Inc()
			return err
		}
		logutil.Logger(ctx).Warn("allegrosql",
			zap.String("label", label),
			zap.Error(err),
			zap.String("txn", s.txn.GoString()))
		ekv.BackOff(retryCnt)
		s.txn.changeToInvalid()
		s.stochastikVars.SetStatusFlag(allegrosql.ServerStatusInTrans, false)
	}
	return err
}

func sqlForLog(allegrosql string) string {
	if len(allegrosql) > sqlLogMaxLen {
		allegrosql = allegrosql[:sqlLogMaxLen] + fmt.Sprintf("(len:%d)", len(allegrosql))
	}
	return interlock.QueryReplacer.Replace(allegrosql)
}

type stochastikPool interface {
	Get() (pools.Resource, error)
	Put(pools.Resource)
}

func (s *stochastik) sysStochastikPool() stochastikPool {
	return petri.GetPetri(s).SysStochastikPool()
}

// InterDircRestrictedALLEGROSQL implements RestrictedALLEGROSQLInterlockingDirectorate interface.
// This is used for executing some restricted allegrosql memexs, usually executed during a normal memex execution.
// Unlike normal InterDirc, it doesn't reset memex status, doesn't commit or rollback the current transaction
// and doesn't write binlog.
func (s *stochastik) InterDircRestrictedALLEGROSQL(allegrosql string) ([]chunk.Row, []*ast.ResultField, error) {
	return s.InterDircRestrictedALLEGROSQLWithContext(context.TODO(), allegrosql)
}

// InterDircRestrictedALLEGROSQLWithContext implements RestrictedALLEGROSQLInterlockingDirectorate interface.
func (s *stochastik) InterDircRestrictedALLEGROSQLWithContext(ctx context.Context, allegrosql string) ([]chunk.Row, []*ast.ResultField, error) {
	// Use special stochastik to execute the allegrosql.
	tmp, err := s.sysStochastikPool().Get()
	if err != nil {
		return nil, nil, err
	}
	se := tmp.(*stochastik)
	// The special stochastik will share the `InspectionTableCache` with current stochastik
	// if the current stochastik in inspection mode.
	if cache := s.stochastikVars.InspectionTableCache; cache != nil {
		se.stochastikVars.InspectionTableCache = cache
		defer func() { se.stochastikVars.InspectionTableCache = nil }()
	}
	if ok := s.stochastikVars.OptimizerUseInvisibleIndexes; ok {
		se.stochastikVars.OptimizerUseInvisibleIndexes = true
		defer func() { se.stochastikVars.OptimizerUseInvisibleIndexes = false }()
	}
	defer func() {
		if se != nil && se.GetStochastikVars().StmtCtx.WarningCount() > 0 {
			warnings := se.GetStochastikVars().StmtCtx.GetWarnings()
			s.GetStochastikVars().StmtCtx.AppendWarnings(warnings)
		}
		s.sysStochastikPool().Put(tmp)
	}()
	metrics.StochastikRestrictedALLEGROSQLCounter.Inc()

	return execRestrictedALLEGROSQL(ctx, se, allegrosql)
}

// InterDircRestrictedALLEGROSQLWithSnapshot implements RestrictedALLEGROSQLInterlockingDirectorate interface.
// This is used for executing some restricted allegrosql memexs with snapshot.
// If current stochastik sets the snapshot timestamp, then execute with this snapshot timestamp.
// Otherwise, execute with the current transaction start timestamp if the transaction is valid.
func (s *stochastik) InterDircRestrictedALLEGROSQLWithSnapshot(allegrosql string) ([]chunk.Row, []*ast.ResultField, error) {
	ctx := context.TODO()

	// Use special stochastik to execute the allegrosql.
	tmp, err := s.sysStochastikPool().Get()
	if err != nil {
		return nil, nil, err
	}
	se := tmp.(*stochastik)
	// The special stochastik will share the `InspectionTableCache` with current stochastik
	// if the current stochastik in inspection mode.
	if cache := s.stochastikVars.InspectionTableCache; cache != nil {
		se.stochastikVars.InspectionTableCache = cache
		defer func() { se.stochastikVars.InspectionTableCache = nil }()
	}
	defer s.sysStochastikPool().Put(tmp)
	metrics.StochastikRestrictedALLEGROSQLCounter.Inc()
	var snapshot uint64
	txn, err := s.Txn(false)
	if err != nil {
		return nil, nil, err
	}
	if txn.Valid() {
		snapshot = s.txn.StartTS()
	}
	if s.stochastikVars.SnapshotTS != 0 {
		snapshot = s.stochastikVars.SnapshotTS
	}
	// Set snapshot.
	if snapshot != 0 {
		se.stochastikVars.SnapshotschemaReplicant, err = petri.GetPetri(s).GetSnapshotSchemaReplicant(snapshot)
		if err != nil {
			return nil, nil, err
		}
		if err := se.stochastikVars.SetSystemVar(variable.MilevaDBSnapshot, strconv.FormatUint(snapshot, 10)); err != nil {
			return nil, nil, err
		}
		defer func() {
			if err := se.stochastikVars.SetSystemVar(variable.MilevaDBSnapshot, ""); err != nil {
				logutil.BgLogger().Error("set milevadbSnapshot error", zap.Error(err))
			}
			se.stochastikVars.SnapshotschemaReplicant = nil
		}()
	}
	if ok := s.stochastikVars.OptimizerUseInvisibleIndexes; ok {
		se.stochastikVars.OptimizerUseInvisibleIndexes = true
		defer func() { se.stochastikVars.OptimizerUseInvisibleIndexes = false }()
	}
	return execRestrictedALLEGROSQL(ctx, se, allegrosql)
}

func execRestrictedALLEGROSQL(ctx context.Context, se *stochastik, allegrosql string) ([]chunk.Row, []*ast.ResultField, error) {
	ctx = context.WithValue(ctx, execdetails.StmtInterDircDetailKey, &execdetails.StmtInterDircDetails{})
	startTime := time.Now()
	recordSets, err := se.InterDircute(ctx, allegrosql)
	if err != nil {
		return nil, nil, err
	}

	var (
		rows   []chunk.Row
		fields []*ast.ResultField
	)
	// InterDircute all recordset, take out the first one as result.
	for i, rs := range recordSets {
		tmp, err := drainRecordSet(ctx, se, rs)
		if err != nil {
			return nil, nil, err
		}
		if err = rs.Close(); err != nil {
			return nil, nil, err
		}

		if i == 0 {
			rows = tmp
			fields = rs.Fields()
		}
	}
	metrics.QueryDurationHistogram.WithLabelValues(metrics.LblInternal).Observe(time.Since(startTime).Seconds())
	return rows, fields, nil
}

func createStochastikFunc(causetstore ekv.CausetStorage) pools.Factory {
	return func() (pools.Resource, error) {
		se, err := createStochastik(causetstore)
		if err != nil {
			return nil, err
		}
		err = variable.SetStochastikSystemVar(se.stochastikVars, variable.AutoCommit, types.NewStringCauset("1"))
		if err != nil {
			return nil, err
		}
		err = variable.SetStochastikSystemVar(se.stochastikVars, variable.MaxInterDircutionTime, types.NewUintCauset(0))
		if err != nil {
			return nil, errors.Trace(err)
		}
		err = variable.SetStochastikSystemVar(se.stochastikVars, variable.MaxAllowedPacket, types.NewStringCauset("67108864"))
		if err != nil {
			return nil, errors.Trace(err)
		}
		se.stochastikVars.CommonGlobalLoaded = true
		se.stochastikVars.InRestrictedALLEGROSQL = true
		return se, nil
	}
}

func createStochastikWithPetriFunc(causetstore ekv.CausetStorage) func(*petri.Petri) (pools.Resource, error) {
	return func(dom *petri.Petri) (pools.Resource, error) {
		se, err := CreateStochastikWithPetri(causetstore, dom)
		if err != nil {
			return nil, err
		}
		err = variable.SetStochastikSystemVar(se.stochastikVars, variable.AutoCommit, types.NewStringCauset("1"))
		if err != nil {
			return nil, err
		}
		err = variable.SetStochastikSystemVar(se.stochastikVars, variable.MaxInterDircutionTime, types.NewUintCauset(0))
		if err != nil {
			return nil, errors.Trace(err)
		}
		se.stochastikVars.CommonGlobalLoaded = true
		se.stochastikVars.InRestrictedALLEGROSQL = true
		return se, nil
	}
}

func drainRecordSet(ctx context.Context, se *stochastik, rs sqlexec.RecordSet) ([]chunk.Row, error) {
	var rows []chunk.Row
	req := rs.NewChunk()
	for {
		err := rs.Next(ctx, req)
		if err != nil || req.NumRows() == 0 {
			return rows, err
		}
		iter := chunk.NewIterator4Chunk(req)
		for r := iter.Begin(); r != iter.End(); r = iter.Next() {
			rows = append(rows, r)
		}
		req = chunk.Renew(req, se.stochastikVars.MaxChunkSize)
	}
}

// getInterDircRet executes restricted allegrosql and the result is one column.
// It returns a string value.
func (s *stochastik) getInterDircRet(ctx stochastikctx.Context, allegrosql string) (string, error) {
	rows, fields, err := s.InterDircRestrictedALLEGROSQL(allegrosql)
	if err != nil {
		return "", err
	}
	if len(rows) == 0 {
		return "", interlock.ErrResultIsEmpty
	}
	d := rows[0].GetCauset(0, &fields[0].DeferredCauset.FieldType)
	value, err := d.ToString()
	if err != nil {
		return "", err
	}
	return value, nil
}

// GetAllSysVars implements GlobalVarAccessor.GetAllSysVars interface.
func (s *stochastik) GetAllSysVars() (map[string]string, error) {
	if s.Value(stochastikctx.Initing) != nil {
		return nil, nil
	}
	allegrosql := `SELECT VARIABLE_NAME, VARIABLE_VALUE FROM %s.%s;`
	allegrosql = fmt.Sprintf(allegrosql, allegrosql.SystemDB, allegrosql.GlobalVariablesTable)
	rows, _, err := s.InterDircRestrictedALLEGROSQL(allegrosql)
	if err != nil {
		return nil, err
	}
	ret := make(map[string]string, len(rows))
	for _, r := range rows {
		k, v := r.GetString(0), r.GetString(1)
		ret[k] = v
	}
	return ret, nil
}

// GetGlobalSysVar implements GlobalVarAccessor.GetGlobalSysVar interface.
func (s *stochastik) GetGlobalSysVar(name string) (string, error) {
	if s.Value(stochastikctx.Initing) != nil {
		// When running bootstrap or upgrade, we should not access global storage.
		return "", nil
	}
	allegrosql := fmt.Sprintf(`SELECT VARIABLE_VALUE FROM %s.%s WHERE VARIABLE_NAME="%s";`,
		allegrosql.SystemDB, allegrosql.GlobalVariablesTable, name)
	sysVar, err := s.getInterDircRet(s, allegrosql)
	if err != nil {
		if interlock.ErrResultIsEmpty.Equal(err) {
			if sv, ok := variable.SysVars[name]; ok {
				return sv.Value, nil
			}
			return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name)
		}
		return "", err
	}
	return sysVar, nil
}

// SetGlobalSysVar implements GlobalVarAccessor.SetGlobalSysVar interface.
func (s *stochastik) SetGlobalSysVar(name, value string) error {
	if name == variable.ALLEGROSQLModeVar {
		value = allegrosql.FormatALLEGROSQLModeStr(value)
		if _, err := allegrosql.GetALLEGROSQLMode(value); err != nil {
			return err
		}
	}
	var sVal string
	var err error
	sVal, err = variable.ValidateSetSystemVar(s.stochastikVars, name, value, variable.ScopeGlobal)
	if err != nil {
		return err
	}
	name = strings.ToLower(name)
	variable.CheckDeprecationSetSystemVar(s.stochastikVars, name)
	allegrosql := fmt.Sprintf(`REPLACE %s.%s VALUES ('%s', '%s');`,
		allegrosql.SystemDB, allegrosql.GlobalVariablesTable, name, sVal)
	_, _, err = s.InterDircRestrictedALLEGROSQL(allegrosql)
	return err
}

func (s *stochastik) ParseALLEGROSQL(ctx context.Context, allegrosql, charset, collation string) ([]ast.StmtNode, []error, error) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.ParseALLEGROSQL", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
	}
	defer trace.StartRegion(ctx, "ParseALLEGROSQL").End()
	s.BerolinaSQL.SetALLEGROSQLMode(s.stochastikVars.ALLEGROSQLMode)
	s.BerolinaSQL.EnableWindowFunc(s.stochastikVars.EnableWindowFunction)
	return s.BerolinaSQL.Parse(allegrosql, charset, collation)
}

func (s *stochastik) SetProcessInfo(allegrosql string, t time.Time, command byte, maxInterDircutionTime uint64) {
	// If command == allegrosql.ComSleep, it means the ALLEGROALLEGROSQL execution is finished. The processinfo is reset to SLEEP.
	// If the ALLEGROALLEGROSQL finished and the stochastik is not in transaction, the current start timestamp need to reset to 0.
	// Otherwise, it should be set to the transaction start timestamp.
	// Why not reset the transaction start timestamp to 0 when transaction committed?
	// Because the select memex and other memexs need this timestamp to read data,
	// after the transaction is committed. e.g. SHOW MASTER STATUS;
	var curTxnStartTS uint64
	if command != allegrosql.ComSleep || s.GetStochastikVars().InTxn() {
		curTxnStartTS = s.stochastikVars.TxnCtx.StartTS
	}
	pi := soliton.ProcessInfo{
		ID:                    s.stochastikVars.ConnectionID,
		EDB:                   s.stochastikVars.CurrentDB,
		Command:               command,
		Causet:                s.currentCauset,
		CausetExplainRows:     causetembedded.GetExplainRowsForCauset(s.currentCauset),
		Time:                  t,
		State:                 s.Status(),
		Info:                  allegrosql,
		CurTxnStartTS:         curTxnStartTS,
		StmtCtx:               s.stochastikVars.StmtCtx,
		StatsInfo:             causetembedded.GetStatsInfo,
		MaxInterDircutionTime: maxInterDircutionTime,
	}
	_, pi.Digest = s.stochastikVars.StmtCtx.ALLEGROSQLDigest()
	if s.stochastikVars.User != nil {
		pi.User = s.stochastikVars.User.Username
		pi.Host = s.stochastikVars.User.Hostname
	}
	s.processInfo.CausetStore(&pi)
}

func (s *stochastik) InterDircuteInternal(ctx context.Context, allegrosql string) (recordSets []sqlexec.RecordSet, err error) {
	origin := s.stochastikVars.InRestrictedALLEGROSQL
	s.stochastikVars.InRestrictedALLEGROSQL = true
	defer func() {
		s.stochastikVars.InRestrictedALLEGROSQL = origin
	}()
	return s.InterDircute(ctx, allegrosql)
}

func (s *stochastik) InterDircute(ctx context.Context, allegrosql string) (recordSets []sqlexec.RecordSet, err error) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.InterDircute", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
		logutil.Eventf(ctx, "execute: %s", allegrosql)
	}

	stmtNodes, err := s.Parse(ctx, allegrosql)
	if err != nil {
		return nil, err
	}
	if len(stmtNodes) != 1 {
		return nil, errors.New("InterDircute() API doesn't support multiple memexs any more")
	}

	rs, err := s.InterDircuteStmt(ctx, stmtNodes[0])
	if err != nil {
		s.stochastikVars.StmtCtx.AppendError(err)
	}
	if rs == nil {
		return nil, err
	}
	return []sqlexec.RecordSet{rs}, err
}

// Parse parses a query string to raw ast.StmtNode.
func (s *stochastik) Parse(ctx context.Context, allegrosql string) ([]ast.StmtNode, error) {
	charsetInfo, collation := s.stochastikVars.GetCharsetInfo()
	parseStartTime := time.Now()
	stmts, warns, err := s.ParseALLEGROSQL(ctx, allegrosql, charsetInfo, collation)
	if err != nil {
		s.rollbackOnError(ctx)

		// Only print log message when this ALLEGROALLEGROSQL is from the user.
		// Mute the warning for internal ALLEGROSQLs.
		if !s.stochastikVars.InRestrictedALLEGROSQL {
			logutil.Logger(ctx).Warn("parse ALLEGROALLEGROSQL failed", zap.Error(err), zap.String("ALLEGROALLEGROSQL", allegrosql))
		}
		return nil, soliton.SyntaxError(err)
	}

	durParse := time.Since(parseStartTime)
	s.GetStochastikVars().DurationParse = durParse
	isInternal := s.isInternal()
	if isInternal {
		stochastikInterDircuteParseDurationInternal.Observe(durParse.Seconds())
	} else {
		stochastikInterDircuteParseDurationGeneral.Observe(durParse.Seconds())
	}
	for _, warn := range warns {
		s.stochastikVars.StmtCtx.AppendWarning(soliton.SyntaxWarn(warn))
	}
	return stmts, nil
}

func (s *stochastik) InterDircuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlexec.RecordSet, error) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.InterDircuteStmt", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
	}

	s.PrepareTxnCtx(ctx)
	err := s.loadCommonGlobalVariablesIfNeeded()
	if err != nil {
		return nil, err
	}

	s.stochastikVars.StartTime = time.Now()

	// Some executions are done in compile stage, so we reset them before compile.
	if err := interlock.ResetContextOfStmt(s, stmtNode); err != nil {
		return nil, err
	}

	// Transform abstract syntax tree to a physical plan(stored in interlock.InterDircStmt).
	compiler := interlock.Compiler{Ctx: s}
	stmt, err := compiler.Compile(ctx, stmtNode)
	if err != nil {
		s.rollbackOnError(ctx)

		// Only print log message when this ALLEGROALLEGROSQL is from the user.
		// Mute the warning for internal ALLEGROSQLs.
		if !s.stochastikVars.InRestrictedALLEGROSQL {
			logutil.Logger(ctx).Warn("compile ALLEGROALLEGROSQL failed", zap.Error(err), zap.String("ALLEGROALLEGROSQL", stmtNode.Text()))
		}
		return nil, err
	}
	durCompile := time.Since(s.stochastikVars.StartTime)
	s.GetStochastikVars().DurationCompile = durCompile
	if s.isInternal() {
		stochastikInterDircuteCompileDurationInternal.Observe(durCompile.Seconds())
	} else {
		stochastikInterDircuteCompileDurationGeneral.Observe(durCompile.Seconds())
	}
	s.currentCauset = stmt.Causet

	// InterDircute the physical plan.
	logStmt(stmt, s.stochastikVars)
	recordSet, err := runStmt(ctx, s, stmt)
	if err != nil {
		if !ekv.ErrKeyExists.Equal(err) {
			logutil.Logger(ctx).Warn("run memex failed",
				zap.Int64("schemaVersion", s.stochastikVars.TxnCtx.SchemaVersion),
				zap.Error(err),
				zap.String("stochastik", s.String()))
		}
		return nil, err
	}
	return recordSet, nil
}

// runStmt executes the sqlexec.Statement and commit or rollback the current transaction.
func runStmt(ctx context.Context, se *stochastik, s sqlexec.Statement) (rs sqlexec.RecordSet, err error) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("stochastik.runStmt", opentracing.ChildOf(span.Context()))
		span1.LogKV("allegrosql", s.OriginText())
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
	}
	se.SetValue(stochastikctx.QueryString, s.OriginText())
	if _, ok := s.(*interlock.InterDircStmt).StmtNode.(ast.DBSNode); ok {
		se.SetValue(stochastikctx.LastInterDircuteDBS, true)
	} else {
		se.ClearValue(stochastikctx.LastInterDircuteDBS)
	}

	sessVars := se.stochastikVars
	// Save origTxnCtx here to avoid it reset in the transaction retry.
	origTxnCtx := sessVars.TxnCtx
	err = se.checkTxnAborted(s)
	if err != nil {
		return nil, err
	}
	rs, err = s.InterDirc(ctx)
	sessVars.TxnCtx.StatementCount++
	if !s.IsReadOnly(sessVars) {
		// All the history should be added here.
		if err == nil && sessVars.TxnCtx.CouldRetry {
			GetHistory(se).Add(s, sessVars.StmtCtx)
		}

		// Handle the stmt commit/rollback.
		if se.txn.Valid() {
			if err != nil {
				se.StmtRollback()
			} else {
				se.StmtCommit()
			}
		}
	}
	if rs != nil {
		return &execStmtResult{
			RecordSet:  rs,
			allegrosql: s,
			se:         se,
		}, err
	}

	err = finishStmt(ctx, se, err, s)

	// If it is not a select memex, we record its slow log here,
	// then it could include the transaction commit time.
	s.(*interlock.InterDircStmt).FinishInterDircuteStmt(origTxnCtx.StartTS, err == nil, false)
	return nil, err
}

// execStmtResult is the return value of InterDircuteStmt and it implements the sqlexec.RecordSet interface.
// Why we need a struct to wrap a RecordSet and provide another RecordSet?
// This is because there are so many stochastik state related things that definitely not belongs to the original
// RecordSet, so this struct exists and RecordSet.Close() is overrided handle that.
type execStmtResult struct {
	sqlexec.RecordSet
	se         *stochastik
	allegrosql sqlexec.Statement
}

func (rs *execStmtResult) Close() error {
	se := rs.se
	err := rs.RecordSet.Close()
	return finishStmt(context.Background(), se, err, rs.allegrosql)
}

// rollbackOnError makes sure the next memex starts a new transaction with the latest SchemaReplicant.
func (s *stochastik) rollbackOnError(ctx context.Context) {
	if !s.stochastikVars.InTxn() {
		s.RollbackTxn(ctx)
	}
}

// PrepareStmt is used for executing prepare memex in binary protocol
func (s *stochastik) PrepareStmt(allegrosql string) (stmtID uint32, paramCount int, fields []*ast.ResultField, err error) {
	if s.stochastikVars.TxnCtx.SchemaReplicant == nil {
		// We don't need to create a transaction for prepare memex, just get information schemaReplicant will do.
		s.stochastikVars.TxnCtx.SchemaReplicant = petri.GetPetri(s).SchemaReplicant()
	}
	err = s.loadCommonGlobalVariablesIfNeeded()
	if err != nil {
		return
	}

	ctx := context.Background()
	inTxn := s.GetStochastikVars().InTxn()
	// NewPrepareInterDirc may need startTS to build the interlock, for example prepare memex has subquery in int.
	// So we have to call PrepareTxnCtx here.
	s.PrepareTxnCtx(ctx)
	s.PrepareTSFuture(ctx)
	prepareInterDirc := interlock.NewPrepareInterDirc(s, schemareplicant.GetSchemaReplicant(s), allegrosql)
	err = prepareInterDirc.Next(ctx, nil)
	if err != nil {
		return
	}
	if !inTxn {
		// We could start a transaction to build the prepare interlock before, we should rollback it here.
		s.RollbackTxn(ctx)
	}
	return prepareInterDirc.ID, prepareInterDirc.ParamCount, prepareInterDirc.Fields, nil
}

func (s *stochastik) preparedStmtInterDirc(ctx context.Context,
	stmtID uint32, prepareStmt *causetembedded.CachedPrepareStmt, args []types.Causet) (sqlexec.RecordSet, error) {
	st, err := interlock.CompileInterDircutePreparedStmt(ctx, s, stmtID, args)
	if err != nil {
		return nil, err
	}
	stochastikInterDircuteCompileDurationGeneral.Observe(time.Since(s.stochastikVars.StartTime).Seconds())
	logQuery(st.OriginText(), s.stochastikVars)
	return runStmt(ctx, s, st)
}

// cachedCausetInterDirc short path currently ONLY for cached "point select plan" execution
func (s *stochastik) cachedCausetInterDirc(ctx context.Context,
	stmtID uint32, prepareStmt *causetembedded.CachedPrepareStmt, args []types.Causet) (sqlexec.RecordSet, error) {
	prepared := prepareStmt.PreparedAst
	// compile InterDircStmt
	is := schemareplicant.GetSchemaReplicant(s)
	execAst := &ast.InterDircuteStmt{InterDircID: stmtID}
	if err := interlock.ResetContextOfStmt(s, execAst); err != nil {
		return nil, err
	}
	execAst.BinaryArgs = args
	execCauset, err := causet.OptimizeInterDircStmt(ctx, s, execAst, is)
	if err != nil {
		return nil, err
	}

	stmtCtx := s.GetStochastikVars().StmtCtx
	stmt := &interlock.InterDircStmt{
		GoCtx:           ctx,
		SchemaReplicant: is,
		Causet:          execCauset,
		StmtNode:        execAst,
		Ctx:             s,
		OutputNames:     execCauset.OutputNames(),
		PsStmt:          prepareStmt,
	}
	compileDuration := time.Since(s.stochastikVars.StartTime)
	stochastikInterDircuteCompileDurationGeneral.Observe(compileDuration.Seconds())
	s.GetStochastikVars().DurationCompile = compileDuration

	stmt.Text = prepared.Stmt.Text()
	stmtCtx.OriginalALLEGROSQL = stmt.Text
	stmtCtx.InitALLEGROSQLDigest(prepareStmt.NormalizedALLEGROSQL, prepareStmt.ALLEGROSQLDigest)
	stmtCtx.SetCausetDigest(prepareStmt.NormalizedCauset, prepareStmt.CausetDigest)
	logQuery(stmt.GetTextToLog(), s.stochastikVars)

	// run InterDircStmt
	var resultSet sqlexec.RecordSet
	switch prepared.CachedCauset.(type) {
	case *causetembedded.PointGetCauset:
		resultSet, err = stmt.PointGet(ctx, is)
		s.txn.changeToInvalid()
	case *causetembedded.UFIDelate:
		s.PrepareTSFuture(ctx)
		stmtCtx.Priority = ekv.PriorityHigh
		resultSet, err = runStmt(ctx, s, stmt)
	default:
		prepared.CachedCauset = nil
		return nil, errors.Errorf("invalid cached plan type")
	}
	return resultSet, err
}

// IsCachedInterDircOk check if we can execute using plan cached in prepared structure
// Be careful for the short path, current precondition is ths cached plan satisfying
// IsPointGetWithPKOrUniqueKeyByAutoCommit
func (s *stochastik) IsCachedInterDircOk(ctx context.Context, preparedStmt *causetembedded.CachedPrepareStmt) (bool, error) {
	prepared := preparedStmt.PreparedAst
	if prepared.CachedCauset == nil {
		return false, nil
	}
	// check auto commit
	if !s.GetStochastikVars().IsAutocommit() {
		return false, nil
	}
	// check schemaReplicant version
	is := schemareplicant.GetSchemaReplicant(s)
	if prepared.SchemaVersion != is.SchemaMetaVersion() {
		prepared.CachedCauset = nil
		return false, nil
	}
	// maybe we'd better check cached plan type here, current
	// only point select/uFIDelate will be cached, see "getPhysicalCauset" func
	var ok bool
	var err error
	switch prepared.CachedCauset.(type) {
	case *causetembedded.PointGetCauset:
		ok = true
	case *causetembedded.UFIDelate:
		pointUFIDelate := prepared.CachedCauset.(*causetembedded.UFIDelate)
		_, ok = pointUFIDelate.SelectCauset.(*causetembedded.PointGetCauset)
		if !ok {
			err = errors.Errorf("cached uFIDelate plan not point uFIDelate")
			prepared.CachedCauset = nil
			return false, err
		}
	default:
		ok = false
	}
	return ok, err
}

// InterDircutePreparedStmt executes a prepared memex.
func (s *stochastik) InterDircutePreparedStmt(ctx context.Context, stmtID uint32, args []types.Causet) (sqlexec.RecordSet, error) {
	s.PrepareTxnCtx(ctx)
	var err error
	s.stochastikVars.StartTime = time.Now()
	preparedPointer, ok := s.stochastikVars.PreparedStmts[stmtID]
	if !ok {
		err = causetembedded.ErrStmtNotFound
		logutil.Logger(ctx).Error("prepared memex not found", zap.Uint32("stmtID", stmtID))
		return nil, err
	}
	preparedStmt, ok := preparedPointer.(*causetembedded.CachedPrepareStmt)
	if !ok {
		return nil, errors.Errorf("invalid CachedPrepareStmt type")
	}
	interlock.CountStmtNode(preparedStmt.PreparedAst.Stmt, s.stochastikVars.InRestrictedALLEGROSQL)
	ok, err = s.IsCachedInterDircOk(ctx, preparedStmt)
	if err != nil {
		return nil, err
	}
	if ok {
		return s.cachedCausetInterDirc(ctx, stmtID, preparedStmt, args)
	}
	return s.preparedStmtInterDirc(ctx, stmtID, preparedStmt, args)
}

func (s *stochastik) DropPreparedStmt(stmtID uint32) error {
	vars := s.stochastikVars
	if _, ok := vars.PreparedStmts[stmtID]; !ok {
		return causetembedded.ErrStmtNotFound
	}
	vars.RetryInfo.DroppedPreparedStmtIDs = append(vars.RetryInfo.DroppedPreparedStmtIDs, stmtID)
	return nil
}

func (s *stochastik) Txn(active bool) (ekv.Transaction, error) {
	if !active {
		return &s.txn, nil
	}
	if !s.txn.validOrPending() {
		return &s.txn, errors.AddStack(ekv.ErrInvalidTxn)
	}
	if s.txn.pending() {
		defer func(begin time.Time) {
			s.stochastikVars.DurationWaitTS = time.Since(begin)
		}(time.Now())
		// Transaction is lazy initialized.
		// PrepareTxnCtx is called to get a tso future, makes s.txn a pending txn,
		// If Txn() is called later, wait for the future to get a valid txn.
		if err := s.txn.changePendingToValid(s.currentCtx); err != nil {
			logutil.BgLogger().Error("active transaction fail",
				zap.Error(err))
			s.txn.cleanup()
			s.stochastikVars.TxnCtx.StartTS = 0
			return &s.txn, err
		}
		s.stochastikVars.TxnCtx.StartTS = s.txn.StartTS()
		if s.stochastikVars.TxnCtx.IsPessimistic {
			s.txn.SetOption(ekv.Pessimistic, true)
		}
		if !s.stochastikVars.IsAutocommit() {
			s.stochastikVars.SetStatusFlag(allegrosql.ServerStatusInTrans, true)
		}
		s.stochastikVars.TxnCtx.CouldRetry = s.isTxnRetryable()
		s.txn.SetVars(s.stochastikVars.KVVars)
		if s.stochastikVars.GetReplicaRead().IsFollowerRead() {
			s.txn.SetOption(ekv.ReplicaRead, ekv.ReplicaReadFollower)
		}
	}
	return &s.txn, nil
}

// isTxnRetryable (if returns true) means the transaction could retry.
// If the transaction is in pessimistic mode, do not retry.
// If the stochastik is already in transaction, enable retry or internal ALLEGROALLEGROSQL could retry.
// If not, the transaction could always retry, because it should be auto committed transaction.
// Anyway the retry limit is 0, the transaction could not retry.
func (s *stochastik) isTxnRetryable() bool {
	sessVars := s.stochastikVars

	// The pessimistic transaction no need to retry.
	if sessVars.TxnCtx.IsPessimistic {
		return false
	}

	// If retry limit is 0, the transaction could not retry.
	if sessVars.RetryLimit == 0 {
		return false
	}

	// If the stochastik is not InTxn, it is an auto-committed transaction.
	// The auto-committed transaction could always retry.
	if !sessVars.InTxn() {
		return true
	}

	// The internal transaction could always retry.
	if sessVars.InRestrictedALLEGROSQL {
		return true
	}

	// If the retry is enabled, the transaction could retry.
	if !sessVars.DisableTxnAutoRetry {
		return true
	}

	return false
}

func (s *stochastik) NewTxn(ctx context.Context) error {
	if s.txn.Valid() {
		txnID := s.txn.StartTS()
		err := s.CommitTxn(ctx)
		if err != nil {
			return err
		}
		vars := s.GetStochastikVars()
		logutil.Logger(ctx).Info("NewTxn() inside a transaction auto commit",
			zap.Int64("schemaVersion", vars.TxnCtx.SchemaVersion),
			zap.Uint64("txnStartTS", txnID))
	}

	txn, err := s.causetstore.Begin()
	if err != nil {
		return err
	}
	txn.SetVars(s.stochastikVars.KVVars)
	if s.GetStochastikVars().GetReplicaRead().IsFollowerRead() {
		txn.SetOption(ekv.ReplicaRead, ekv.ReplicaReadFollower)
	}
	s.txn.changeInvalidToValid(txn)
	is := petri.GetPetri(s).SchemaReplicant()
	s.stochastikVars.TxnCtx = &variable.TransactionContext{
		SchemaReplicant: is,
		SchemaVersion:   is.SchemaMetaVersion(),
		CreateTime:      time.Now(),
		StartTS:         txn.StartTS(),
		ShardStep:       int(s.stochastikVars.ShardAllocateStep),
	}
	return nil
}

func (s *stochastik) SetValue(key fmt.Stringer, value interface{}) {
	s.mu.Lock()
	s.mu.values[key] = value
	s.mu.Unlock()
}

func (s *stochastik) Value(key fmt.Stringer) interface{} {
	s.mu.RLock()
	value := s.mu.values[key]
	s.mu.RUnlock()
	return value
}

func (s *stochastik) ClearValue(key fmt.Stringer) {
	s.mu.Lock()
	delete(s.mu.values, key)
	s.mu.Unlock()
}

type inCloseStochastik struct{}

// Close function does some clean work when stochastik end.
// Close should release the causet locks which hold by the stochastik.
func (s *stochastik) Close() {
	// TODO: do clean causet locks when stochastik exited without execute Close.
	// TODO: do clean causet locks when milevadb-server was `kill -9`.
	if s.HasLockedTables() && config.TableLockEnabled() {
		if ds := config.TableLockDelayClean(); ds > 0 {
			time.Sleep(time.Duration(ds) * time.Millisecond)
		}
		lockedTables := s.GetAllTableLocks()
		err := petri.GetPetri(s).DBS().UnlockTables(s, lockedTables)
		if err != nil {
			logutil.BgLogger().Error("release causet dagger failed", zap.Uint64("conn", s.stochastikVars.ConnectionID))
		}
	}
	if s.statsDefCauslector != nil {
		s.statsDefCauslector.Delete()
	}
	bindValue := s.Value(bindinfo.StochastikBindInfoKeyType)
	if bindValue != nil {
		bindValue.(*bindinfo.StochastikHandle).Close()
	}
	ctx := context.WithValue(context.TODO(), inCloseStochastik{}, struct{}{})
	s.RollbackTxn(ctx)
	if s.stochastikVars != nil {
		s.stochastikVars.WithdrawAllPreparedStmt()
	}
}

// GetStochastikVars implements the context.Context interface.
func (s *stochastik) GetStochastikVars() *variable.StochastikVars {
	return s.stochastikVars
}

func (s *stochastik) Auth(user *auth.UserIdentity, authentication []byte, salt []byte) bool {
	pm := privilege.GetPrivilegeManager(s)

	// Check IP or localhost.
	var success bool
	user.AuthUsername, user.AuthHostname, success = pm.ConnectionVerification(user.Username, user.Hostname, authentication, salt, s.stochastikVars.TLSConnectionState)
	if success {
		s.stochastikVars.User = user
		s.stochastikVars.ActiveRoles = pm.GetDefaultRoles(user.AuthUsername, user.AuthHostname)
		return true
	} else if user.Hostname == variable.DefHostname {
		return false
	}

	// Check Hostname.
	for _, addr := range getHostByIP(user.Hostname) {
		u, h, success := pm.ConnectionVerification(user.Username, addr, authentication, salt, s.stochastikVars.TLSConnectionState)
		if success {
			s.stochastikVars.User = &auth.UserIdentity{
				Username:     user.Username,
				Hostname:     addr,
				AuthUsername: u,
				AuthHostname: h,
			}
			s.stochastikVars.ActiveRoles = pm.GetDefaultRoles(u, h)
			return true
		}
	}
	return false
}

// AuthWithoutVerification is required by the ResetConnection RPC
func (s *stochastik) AuthWithoutVerification(user *auth.UserIdentity) bool {
	pm := privilege.GetPrivilegeManager(s)

	// Check IP or localhost.
	var success bool
	user.AuthUsername, user.AuthHostname, success = pm.GetAuthWithoutVerification(user.Username, user.Hostname)
	if success {
		s.stochastikVars.User = user
		s.stochastikVars.ActiveRoles = pm.GetDefaultRoles(user.AuthUsername, user.AuthHostname)
		return true
	} else if user.Hostname == variable.DefHostname {
		return false
	}

	// Check Hostname.
	for _, addr := range getHostByIP(user.Hostname) {
		u, h, success := pm.GetAuthWithoutVerification(user.Username, addr)
		if success {
			s.stochastikVars.User = &auth.UserIdentity{
				Username:     user.Username,
				Hostname:     addr,
				AuthUsername: u,
				AuthHostname: h,
			}
			s.stochastikVars.ActiveRoles = pm.GetDefaultRoles(u, h)
			return true
		}
	}
	return false
}

func getHostByIP(ip string) []string {
	if ip == "127.0.0.1" {
		return []string{variable.DefHostname}
	}
	addrs, err := net.LookupAddr(ip)
	if err != nil {
		// The error is ignorable.
		// The empty line here makes the golint tool (which complains err is not checked) happy.
	}
	return addrs
}

// CreateStochastik4Test creates a new stochastik environment for test.
func CreateStochastik4Test(causetstore ekv.CausetStorage) (Stochastik, error) {
	return CreateStochastik4TestWithOpt(causetstore, nil)
}

// Opt describes the option for creating stochastik
type Opt struct {
	PreparedCausetCache *ekvcache.SimpleLRUCache
}

// CreateStochastik4TestWithOpt creates a new stochastik environment for test.
func CreateStochastik4TestWithOpt(causetstore ekv.CausetStorage, opt *Opt) (Stochastik, error) {
	s, err := CreateStochastikWithOpt(causetstore, opt)
	if err == nil {
		// initialize stochastik variables for test.
		s.GetStochastikVars().InitChunkSize = 2
		s.GetStochastikVars().MaxChunkSize = 32
	}
	return s, err
}

// CreateStochastik creates a new stochastik environment.
func CreateStochastik(causetstore ekv.CausetStorage) (Stochastik, error) {
	return CreateStochastikWithOpt(causetstore, nil)
}

// CreateStochastikWithOpt creates a new stochastik environment with option.
// Use default option if opt is nil.
func CreateStochastikWithOpt(causetstore ekv.CausetStorage, opt *Opt) (Stochastik, error) {
	s, err := createStochastikWithOpt(causetstore, opt)
	if err != nil {
		return nil, err
	}

	// Add auth here.
	do, err := domap.Get(causetstore)
	if err != nil {
		return nil, err
	}
	pm := &privileges.UserPrivileges{
		Handle: do.PrivilegeHandle(),
	}
	privilege.BindPrivilegeManager(s, pm)

	stochastikBindHandle := bindinfo.NewStochastikBindHandle(s.BerolinaSQL)
	s.SetValue(bindinfo.StochastikBindInfoKeyType, stochastikBindHandle)
	// Add stats collector, and it will be freed by background stats worker
	// which periodically uFIDelates stats using the collected data.
	if do.StatsHandle() != nil && do.StatsUFIDelating() {
		s.statsDefCauslector = do.StatsHandle().NewStochastikStatsDefCauslector()
	}

	return s, nil
}

// loadSystemTZ loads systemTZ from allegrosql.milevadb
func loadSystemTZ(se *stochastik) (string, error) {
	return loadParameter(se, "system_tz")
}

// loadDefCauslationParameter loads collation parameter from allegrosql.milevadb
func loadDefCauslationParameter(se *stochastik) (bool, error) {
	para, err := loadParameter(se, milevadbNewDefCauslationEnabled)
	if err != nil {
		return false, err
	}
	if para == varTrue {
		return true, nil
	} else if para == varFalse {
		return false, nil
	}
	logutil.BgLogger().Warn(
		"Unexpected value of 'new_collation_enabled' in 'allegrosql.milevadb', use 'False' instead",
		zap.String("value", para))
	return false, nil
}

// loadParameter loads read-only parameter from allegrosql.milevadb
func loadParameter(se *stochastik, name string) (string, error) {
	allegrosql := "select variable_value from allegrosql.milevadb where variable_name = '" + name + "'"
	rss, errLoad := se.InterDircute(context.Background(), allegrosql)
	if errLoad != nil {
		return "", errLoad
	}
	// the record of allegrosql.milevadb under where condition: variable_name = $name should shall only be one.
	defer func() {
		if err := rss[0].Close(); err != nil {
			logutil.BgLogger().Error("close result set error", zap.Error(err))
		}
	}()
	req := rss[0].NewChunk()
	if err := rss[0].Next(context.Background(), req); err != nil {
		return "", err
	}
	return req.GetRow(0).GetString(0), nil
}

// BootstrapStochastik runs the first time when the MilevaDB server start.
func BootstrapStochastik(causetstore ekv.CausetStorage) (*petri.Petri, error) {
	cfg := config.GetGlobalConfig()
	if len(cfg.Plugin.Load) > 0 {
		err := plugin.Load(context.Background(), plugin.Config{
			Plugins:        strings.Split(cfg.Plugin.Load, ","),
			PluginDir:      cfg.Plugin.Dir,
			GlobalSysVar:   &variable.SysVars,
			PluginVarNames: &variable.PluginVarNames,
		})
		if err != nil {
			return nil, err
		}
	}

	initLoadCommonGlobalVarsALLEGROSQL()

	ver := getStoreBootstrapVersion(causetstore)
	if ver == notBootstrapped {
		runInBootstrapStochastik(causetstore, bootstrap)
	} else if ver < currentBootstrapVersion {
		runInBootstrapStochastik(causetstore, upgrade)
	}

	se, err := createStochastik(causetstore)
	if err != nil {
		return nil, err
	}
	// get system tz from allegrosql.milevadb
	tz, err := loadSystemTZ(se)
	if err != nil {
		return nil, err
	}
	timeutil.SetSystemTZ(tz)

	// get the flag from `allegrosql`.`milevadb` which indicating if new collations are enabled.
	newDefCauslationEnabled, err := loadDefCauslationParameter(se)
	if err != nil {
		return nil, err
	}

	if newDefCauslationEnabled {
		collate.EnableNewDefCauslations()
	}

	dom := petri.GetPetri(se)
	dom.InitExpensiveQueryHandle()

	se2, err := createStochastik(causetstore)
	if err != nil {
		return nil, err
	}
	se3, err := createStochastik(causetstore)
	if err != nil {
		return nil, err
	}
	// We should make the load bind-info loop before other loops which has internal ALLEGROALLEGROSQL.
	// Because the internal ALLEGROALLEGROSQL may access the global bind-info handler. As the result, the data race occurs here as the
	// LoadBindInfoLoop inits global bind-info handler.
	err = dom.LoadBindInfoLoop(se2, se3)
	if err != nil {
		return nil, err
	}

	if !config.GetGlobalConfig().Security.SkipGrantTable {
		err = dom.LoadPrivilegeLoop(se)
		if err != nil {
			return nil, err
		}
	}

	if len(cfg.Plugin.Load) > 0 {
		err := plugin.Init(context.Background(), plugin.Config{EtcdClient: dom.GetEtcdClient()})
		if err != nil {
			return nil, err
		}
	}

	err = interlock.LoadExprPushdownBlacklist(se)
	if err != nil {
		return nil, err
	}

	err = interlock.LoadOptMemruleBlacklist(se)
	if err != nil {
		return nil, err
	}

	dom.TelemetryLoop(se)

	se1, err := createStochastik(causetstore)
	if err != nil {
		return nil, err
	}
	err = dom.UFIDelateTableStatsLoop(se1)
	if err != nil {
		return nil, err
	}
	if raw, ok := causetstore.(einsteindb.EtcdBackend); ok {
		err = raw.StartGCWorker()
		if err != nil {
			return nil, err
		}
	}

	return dom, err
}

// GetPetri gets the associated petri for causetstore.
func GetPetri(causetstore ekv.CausetStorage) (*petri.Petri, error) {
	return domap.Get(causetstore)
}

// runInBootstrapStochastik create a special stochastik for bootstrap to run.
// If no bootstrap and storage is remote, we must use a little lease time to
// bootstrap quickly, after bootstrapped, we will reset the lease time.
// TODO: Using a bootstrap tool for doing this may be better later.
func runInBootstrapStochastik(causetstore ekv.CausetStorage, bootstrap func(Stochastik)) {
	s, err := createStochastik(causetstore)
	if err != nil {
		// Bootstrap fail will cause program exit.
		logutil.BgLogger().Fatal("createStochastik error", zap.Error(err))
	}

	s.SetValue(stochastikctx.Initing, true)
	bootstrap(s)
	finishBootstrap(causetstore)
	s.ClearValue(stochastikctx.Initing)

	dom := petri.GetPetri(s)
	dom.Close()
	domap.Delete(causetstore)
}

func createStochastik(causetstore ekv.CausetStorage) (*stochastik, error) {
	return createStochastikWithOpt(causetstore, nil)
}

func createStochastikWithOpt(causetstore ekv.CausetStorage, opt *Opt) (*stochastik, error) {
	dom, err := domap.Get(causetstore)
	if err != nil {
		return nil, err
	}
	s := &stochastik{
		causetstore:      causetstore,
		BerolinaSQL:      BerolinaSQL.New(),
		stochastikVars:   variable.NewStochastikVars(),
		dbsTenantChecker: dom.DBS().TenantManager(),
		client:           causetstore.GetClient(),
	}
	if causetembedded.PreparedCausetCacheEnabled() {
		if opt != nil && opt.PreparedCausetCache != nil {
			s.preparedCausetCache = opt.PreparedCausetCache
		} else {
			s.preparedCausetCache = ekvcache.NewSimpleLRUCache(causetembedded.PreparedCausetCacheCapacity,
				causetembedded.PreparedCausetCacheMemoryGuardRatio, causetembedded.PreparedCausetCacheMaxMemory.Load())
		}
	}
	s.mu.values = make(map[fmt.Stringer]interface{})
	s.lockedTables = make(map[int64]perceptron.TableLockTpInfo)
	petri.BindPetri(s, dom)
	// stochastik implements variable.GlobalVarAccessor. Bind it to ctx.
	s.stochastikVars.GlobalVarsAccessor = s
	s.stochastikVars.BinlogClient = binloginfo.GetPumpsClient()
	s.txn.init()

	stochastikBindHandle := bindinfo.NewStochastikBindHandle(s.BerolinaSQL)
	s.SetValue(bindinfo.StochastikBindInfoKeyType, stochastikBindHandle)
	return s, nil
}

// CreateStochastikWithPetri creates a new Stochastik and binds it with a Petri.
// We need this because when we start DBS in Petri, the DBS need a stochastik
// to change some system blocks. But at that time, we have been already in
// a dagger context, which cause we can't call createSesion directly.
func CreateStochastikWithPetri(causetstore ekv.CausetStorage, dom *petri.Petri) (*stochastik, error) {
	s := &stochastik{
		causetstore:    causetstore,
		BerolinaSQL:    BerolinaSQL.New(),
		stochastikVars: variable.NewStochastikVars(),
		client:         causetstore.GetClient(),
	}
	if causetembedded.PreparedCausetCacheEnabled() {
		s.preparedCausetCache = ekvcache.NewSimpleLRUCache(causetembedded.PreparedCausetCacheCapacity,
			causetembedded.PreparedCausetCacheMemoryGuardRatio, causetembedded.PreparedCausetCacheMaxMemory.Load())
	}
	s.mu.values = make(map[fmt.Stringer]interface{})
	s.lockedTables = make(map[int64]perceptron.TableLockTpInfo)
	petri.BindPetri(s, dom)
	// stochastik implements variable.GlobalVarAccessor. Bind it to ctx.
	s.stochastikVars.GlobalVarsAccessor = s
	s.txn.init()
	return s, nil
}

const (
	notBootstrapped         = 0
	currentBootstrapVersion = version51
)

func getStoreBootstrapVersion(causetstore ekv.CausetStorage) int64 {
	storeBootstrappedLock.Lock()
	defer storeBootstrappedLock.Unlock()
	// check in memory
	_, ok := storeBootstrapped[causetstore.UUID()]
	if ok {
		return currentBootstrapVersion
	}

	var ver int64
	// check in ekv causetstore
	err := ekv.RunInNewTxn(causetstore, false, func(txn ekv.Transaction) error {
		var err error
		t := spacetime.NewMeta(txn)
		ver, err = t.GetBootstrapVersion()
		return err
	})

	if err != nil {
		logutil.BgLogger().Fatal("check bootstrapped failed",
			zap.Error(err))
	}

	if ver > notBootstrapped {
		// here mean memory is not ok, but other server has already finished it
		storeBootstrapped[causetstore.UUID()] = true
	}

	return ver
}

func finishBootstrap(causetstore ekv.CausetStorage) {
	setStoreBootstrapped(causetstore.UUID())

	err := ekv.RunInNewTxn(causetstore, true, func(txn ekv.Transaction) error {
		t := spacetime.NewMeta(txn)
		err := t.FinishBootstrap(currentBootstrapVersion)
		return err
	})
	if err != nil {
		logutil.BgLogger().Fatal("finish bootstrap failed",
			zap.Error(err))
	}
}

const quoteCommaQuote = "', '"

var builtinGlobalVariable = []string{
	variable.AutoCommit,
	variable.ALLEGROSQLModeVar,
	variable.MaxAllowedPacket,
	variable.TimeZone,
	variable.BlockEncryptionMode,
	variable.WaitTimeout,
	variable.InteractiveTimeout,
	variable.MaxPreparedStmtCount,
	variable.InitConnect,
	variable.TxnIsolation,
	variable.TxReadOnly,
	variable.TransactionIsolation,
	variable.TransactionReadOnly,
	variable.NetBufferLength,
	variable.QueryCacheType,
	variable.QueryCacheSize,
	variable.CharacterSetServer,
	variable.AutoIncrementIncrement,
	variable.AutoIncrementOffset,
	variable.DefCauslationServer,
	variable.NetWriteTimeout,
	variable.MaxInterDircutionTime,
	variable.InnodbLockWaitTimeout,
	variable.WindowingUseHighPrecision,
	variable.ALLEGROSQLSelectLimit,

	/* MilevaDB specific global variables: */
	variable.MilevaDBSkipASCIICheck,
	variable.MilevaDBSkipUTF8Check,
	variable.MilevaDBIndexJoinBatchSize,
	variable.MilevaDBIndexLookupSize,
	variable.MilevaDBIndexLookupConcurrency,
	variable.MilevaDBIndexLookupJoinConcurrency,
	variable.MilevaDBIndexSerialScanConcurrency,
	variable.MilevaDBHashJoinConcurrency,
	variable.MilevaDBProjectionConcurrency,
	variable.MilevaDBHashAggPartialConcurrency,
	variable.MilevaDBHashAggFinalConcurrency,
	variable.MilevaDBWindowConcurrency,
	variable.MilevaDBInterlockingDirectorateConcurrency,
	variable.MilevaDBBackoffLockFast,
	variable.MilevaDBBackOffWeight,
	variable.MilevaDBConstraintCheckInPlace,
	variable.MilevaDBDBSReorgWorkerCount,
	variable.MilevaDBDBSReorgBatchSize,
	variable.MilevaDBDBSErrorCountLimit,
	variable.MilevaDBOptInSubqToJoinAnPosetDagg,
	variable.MilevaDBOptCorrelationThreshold,
	variable.MilevaDBOptCorrelationExpFactor,
	variable.MilevaDBOptCPUFactor,
	variable.MilevaDBOptCopCPUFactor,
	variable.MilevaDBOptNetworkFactor,
	variable.MilevaDBOptScanFactor,
	variable.MilevaDBOptDescScanFactor,
	variable.MilevaDBOptMemoryFactor,
	variable.MilevaDBOptDiskFactor,
	variable.MilevaDBOptConcurrencyFactor,
	variable.MilevaDBDistALLEGROSQLScanConcurrency,
	variable.MilevaDBInitChunkSize,
	variable.MilevaDBMaxChunkSize,
	variable.MilevaDBEnableCascadesCausetAppend,
	variable.MilevaDBRetryLimit,
	variable.MilevaDBDisableTxnAutoRetry,
	variable.MilevaDBEnableWindowFunction,
	variable.MilevaDBEnableTablePartition,
	variable.MilevaDBEnableVectorizedExpression,
	variable.MilevaDBEnableFastAnalyze,
	variable.MilevaDBExpensiveQueryTimeThreshold,
	variable.MilevaDBEnableNoopFuncs,
	variable.MilevaDBEnableIndexMerge,
	variable.MilevaDBTxnMode,
	variable.MilevaDBAllowBatchCop,
	variable.MilevaDBOptBCJ,
	variable.MilevaDBRowFormatVersion,
	variable.MilevaDBEnableStmtSummary,
	variable.MilevaDBStmtSummaryInternalQuery,
	variable.MilevaDBStmtSummaryRefreshInterval,
	variable.MilevaDBStmtSummaryHistorySize,
	variable.MilevaDBStmtSummaryMaxStmtCount,
	variable.MilevaDBStmtSummaryMaxALLEGROSQLLength,
	variable.MilevaDBMaxDeltaSchemaCount,
	variable.MilevaDBCaptureCausetBaseline,
	variable.MilevaDBUseCausetBaselines,
	variable.MilevaDBEvolveCausetBaselines,
	variable.MilevaDBIsolationReadEngines,
	variable.MilevaDBStoreLimit,
	variable.MilevaDBAllowAutoRandExplicitInsert,
	variable.MilevaDBEnableClusteredIndex,
	variable.MilevaDBPartitionPruneMode,
	variable.MilevaDBSlowLogMasking,
	variable.MilevaDBRedactLog,
	variable.MilevaDBEnableTelemetry,
	variable.MilevaDBShardAllocateStep,
	variable.MilevaDBEnableChangeDeferredCausetType,
	variable.MilevaDBEnableAmendPessimisticTxn,
}

var (
	loadCommonGlobalVarsALLEGROSQLOnce sync.Once
	loadCommonGlobalVarsALLEGROSQL     string
)

func initLoadCommonGlobalVarsALLEGROSQL() {
	loadCommonGlobalVarsALLEGROSQLOnce.Do(func() {
		vars := append(make([]string, 0, len(builtinGlobalVariable)+len(variable.PluginVarNames)), builtinGlobalVariable...)
		if len(variable.PluginVarNames) > 0 {
			vars = append(vars, variable.PluginVarNames...)
		}
		loadCommonGlobalVarsALLEGROSQL = "select HIGH_PRIORITY * from allegrosql.global_variables where variable_name in ('" + strings.Join(vars, quoteCommaQuote) + "')"
	})
}

// loadCommonGlobalVariablesIfNeeded loads and applies commonly used global variables for the stochastik.
func (s *stochastik) loadCommonGlobalVariablesIfNeeded() error {
	initLoadCommonGlobalVarsALLEGROSQL()
	vars := s.stochastikVars
	if vars.CommonGlobalLoaded {
		return nil
	}
	if s.Value(stochastikctx.Initing) != nil {
		// When running bootstrap or upgrade, we should not access global storage.
		return nil
	}

	var err error
	// Use GlobalVariableCache if MilevaDB just loaded global variables within 2 second ago.
	// When a lot of connections connect to MilevaDB simultaneously, it can protect EinsteinDB spacetime region from overload.
	gvc := petri.GetPetri(s).GetGlobalVarsCache()
	loadFunc := func() ([]chunk.Row, []*ast.ResultField, error) {
		return s.InterDircRestrictedALLEGROSQL(loadCommonGlobalVarsALLEGROSQL)
	}
	rows, fields, err := gvc.LoadGlobalVariables(loadFunc)
	if err != nil {
		logutil.BgLogger().Warn("failed to load global variables",
			zap.Uint64("conn", s.stochastikVars.ConnectionID), zap.Error(err))
		return err
	}
	vars.CommonGlobalLoaded = true

	for _, event := range rows {
		varName := event.GetString(0)
		varVal := event.GetCauset(1, &fields[1].DeferredCauset.FieldType)
		if _, ok := vars.GetSystemVar(varName); !ok {
			err = variable.SetStochastikSystemVar(s.stochastikVars, varName, varVal)
			if err != nil {
				return err
			}
		}
	}

	// when client set Capability Flags CLIENT_INTERACTIVE, init wait_timeout with interactive_timeout
	if vars.ClientCapability&allegrosql.ClientInteractive > 0 {
		if varVal, ok := vars.GetSystemVar(variable.InteractiveTimeout); ok {
			if err := vars.SetSystemVar(variable.WaitTimeout, varVal); err != nil {
				return err
			}
		}
	}

	vars.CommonGlobalLoaded = true
	return nil
}

// PrepareTxnCtx starts a goroutine to begin a transaction if needed, and creates a new transaction context.
// It is called before we execute a allegrosql query.
func (s *stochastik) PrepareTxnCtx(ctx context.Context) {
	s.currentCtx = ctx
	if s.txn.validOrPending() {
		return
	}

	is := petri.GetPetri(s).SchemaReplicant()
	s.stochastikVars.TxnCtx = &variable.TransactionContext{
		SchemaReplicant: is,
		SchemaVersion:   is.SchemaMetaVersion(),
		CreateTime:      time.Now(),
		ShardStep:       int(s.stochastikVars.ShardAllocateStep),
	}
	if !s.stochastikVars.IsAutocommit() || s.stochastikVars.RetryInfo.Retrying {
		if s.stochastikVars.TxnMode == ast.Pessimistic {
			s.stochastikVars.TxnCtx.IsPessimistic = true
		}
	}
}

// PrepareTSFuture uses to try to get ts future.
func (s *stochastik) PrepareTSFuture(ctx context.Context) {
	if !s.txn.validOrPending() {
		// Prepare the transaction future if the transaction is invalid (at the beginning of the transaction).
		txnFuture := s.getTxnFuture(ctx)
		s.txn.changeInvalidToPending(txnFuture)
	} else if s.txn.Valid() && s.GetStochastikVars().IsPessimisticReadConsistency() {
		// Prepare the memex future if the transaction is valid in RC transactions.
		s.GetStochastikVars().TxnCtx.SetStmtFutureForRC(s.getTxnFuture(ctx).future)
	}
}

// RefreshTxnCtx implements context.RefreshTxnCtx interface.
func (s *stochastik) RefreshTxnCtx(ctx context.Context) error {
	if err := s.doCommit(ctx); err != nil {
		return err
	}

	return s.NewTxn(ctx)
}

// InitTxnWithStartTS create a transaction with startTS.
func (s *stochastik) InitTxnWithStartTS(startTS uint64) error {
	if s.txn.Valid() {
		return nil
	}

	// no need to get txn from txnFutureCh since txn should init with startTs
	txn, err := s.causetstore.BeginWithStartTS(startTS)
	if err != nil {
		return err
	}
	txn.SetVars(s.stochastikVars.KVVars)
	s.txn.changeInvalidToValid(txn)
	err = s.loadCommonGlobalVariablesIfNeeded()
	if err != nil {
		return err
	}
	return nil
}

// GetStore gets the causetstore of stochastik.
func (s *stochastik) GetStore() ekv.CausetStorage {
	return s.causetstore
}

func (s *stochastik) ShowProcess() *soliton.ProcessInfo {
	var pi *soliton.ProcessInfo
	tmp := s.processInfo.Load()
	if tmp != nil {
		pi = tmp.(*soliton.ProcessInfo)
	}
	return pi
}

// logStmt logs some crucial ALLEGROALLEGROSQL including: CREATE USER/GRANT PRIVILEGE/CHANGE PASSWORD/DBS etc and normal ALLEGROALLEGROSQL
// if variable.ProcessGeneralLog is set.
func logStmt(execStmt *interlock.InterDircStmt, vars *variable.StochastikVars) {
	switch stmt := execStmt.StmtNode.(type) {
	case *ast.CreateUserStmt, *ast.DropUserStmt, *ast.AlterUserStmt, *ast.SetPwdStmt, *ast.GrantStmt,
		*ast.RevokeStmt, *ast.AlterTableStmt, *ast.CreateDatabaseStmt, *ast.CreateIndexStmt, *ast.CreateTableStmt,
		*ast.DroFIDelatabaseStmt, *ast.DropIndexStmt, *ast.DropTableStmt, *ast.RenameTableStmt, *ast.TruncateTableStmt:
		user := vars.User
		schemaVersion := vars.TxnCtx.SchemaVersion
		if ss, ok := execStmt.StmtNode.(ast.SensitiveStmtNode); ok {
			logutil.BgLogger().Info("CRUCIAL OPERATION",
				zap.Uint64("conn", vars.ConnectionID),
				zap.Int64("schemaVersion", schemaVersion),
				zap.String("secure text", ss.SecureText()),
				zap.Stringer("user", user))
		} else {
			logutil.BgLogger().Info("CRUCIAL OPERATION",
				zap.Uint64("conn", vars.ConnectionID),
				zap.Int64("schemaVersion", schemaVersion),
				zap.String("cur_db", vars.CurrentDB),
				zap.String("allegrosql", stmt.Text()),
				zap.Stringer("user", user))
		}
	default:
		logQuery(execStmt.GetTextToLog(), vars)
	}
}

func logQuery(query string, vars *variable.StochastikVars) {
	if atomic.LoadUint32(&variable.ProcessGeneralLog) != 0 && !vars.InRestrictedALLEGROSQL {
		query = interlock.QueryReplacer.Replace(query)
		if !config.RedactLogEnabled() {
			query = query + vars.PreparedParams.String()
		}
		logutil.BgLogger().Info("GENERAL_LOG",
			zap.Uint64("conn", vars.ConnectionID),
			zap.Stringer("user", vars.User),
			zap.Int64("schemaVersion", vars.TxnCtx.SchemaVersion),
			zap.Uint64("txnStartTS", vars.TxnCtx.StartTS),
			zap.Uint64("forUFIDelateTS", vars.TxnCtx.GetForUFIDelateTS()),
			zap.Bool("isReadConsistency", vars.IsReadConsistencyTxn()),
			zap.String("current_db", vars.CurrentDB),
			zap.String("txn_mode", vars.GetReadableTxnMode()),
			zap.String("allegrosql", query))
	}
}

func (s *stochastik) recordOnTransactionInterDircution(err error, counter int, duration float64) {
	if s.stochastikVars.TxnCtx.IsPessimistic {
		if err != nil {
			memexPerTransactionPessimisticError.Observe(float64(counter))
			transactionDurationPessimisticAbort.Observe(duration)
		} else {
			memexPerTransactionPessimisticOK.Observe(float64(counter))
			transactionDurationPessimisticCommit.Observe(duration)
		}
	} else {
		if err != nil {
			memexPerTransactionOptimisticError.Observe(float64(counter))
			transactionDurationOptimisticAbort.Observe(duration)
		} else {
			memexPerTransactionOptimisticOK.Observe(float64(counter))
			transactionDurationOptimisticCommit.Observe(duration)
		}
	}
}