github.com/matrixorigin/matrixone@v1.2.0/pkg/util/trace/impl/motrace/report_statement.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package motrace
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"errors"
    21  	"fmt"
    22  	"strings"
    23  	"sync"
    24  	"time"
    25  	"unsafe"
    26  
    27  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    28  	"github.com/matrixorigin/matrixone/pkg/common/util"
    29  	"github.com/matrixorigin/matrixone/pkg/container/types"
    30  	"github.com/matrixorigin/matrixone/pkg/frontend/constant"
    31  	"github.com/matrixorigin/matrixone/pkg/logutil"
    32  	db_holder "github.com/matrixorigin/matrixone/pkg/util/export/etl/db"
    33  	"github.com/matrixorigin/matrixone/pkg/util/export/table"
    34  	"github.com/matrixorigin/matrixone/pkg/util/metric"
    35  	v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2"
    36  	"github.com/matrixorigin/matrixone/pkg/util/trace/impl/motrace/statistic"
    37  
    38  	"github.com/google/uuid"
    39  	"go.uber.org/zap"
    40  )
    41  
    42  var NilStmtID [16]byte
    43  var NilTxnID [16]byte
    44  var NilSesID [16]byte
    45  
    46  // StatementInfo implement export.IBuffer2SqlItem and export.CsvFields
    47  
    48  var _ IBuffer2SqlItem = (*StatementInfo)(nil)
    49  
    50  const Decimal128Width = 38
    51  const Decimal128Scale = 0
    52  
    53  func convertFloat64ToDecimal128(val float64) (types.Decimal128, error) {
    54  	return types.Decimal128FromFloat64(val, Decimal128Width, Decimal128Scale)
    55  }
    56  func mustDecimal128(v types.Decimal128, err error) types.Decimal128 {
    57  	if err != nil {
    58  		logutil.Panic("mustDecimal128", zap.Error(err))
    59  	}
    60  	return v
    61  }
    62  
    63  func StatementInfoNew(i Item, ctx context.Context) Item {
    64  	stmt := NewStatementInfo() // Get a new statement from the pool
    65  	if s, ok := i.(*StatementInfo); ok {
    66  
    67  		// execute the stat plan
    68  		s.ExecPlan2Stats(ctx)
    69  
    70  		// remove the plan, s will be free
    71  		s.jsonByte = nil
    72  		s.FreeExecPlan()
    73  		s.exported = true
    74  
    75  		// copy value
    76  		stmt.StatementID = s.StatementID
    77  		stmt.SessionID = s.SessionID
    78  		stmt.TransactionID = s.TransactionID
    79  		stmt.Account = s.Account
    80  		stmt.User = s.User
    81  		stmt.Host = s.Host
    82  		stmt.RoleId = s.RoleId
    83  		stmt.StatementType = s.StatementType
    84  		stmt.QueryType = s.QueryType
    85  		stmt.SqlSourceType = s.SqlSourceType
    86  		stmt.Statement = s.Statement
    87  		stmt.StmtBuilder.WriteString(s.Statement)
    88  		stmt.Status = s.Status
    89  		stmt.Duration = s.Duration
    90  		stmt.ResultCount = s.ResultCount
    91  		stmt.RowsRead = s.RowsRead
    92  		stmt.BytesScan = s.BytesScan
    93  		stmt.ConnType = s.ConnType
    94  		stmt.statsArray = s.statsArray
    95  		stmt.RequestAt = s.RequestAt
    96  		stmt.ResponseAt = s.ResponseAt
    97  		stmt.end = s.end
    98  		stmt.StatementTag = s.StatementTag
    99  		stmt.StatementFingerprint = s.StatementFingerprint
   100  		stmt.Error = s.Error
   101  		stmt.Database = s.Database
   102  		stmt.AggrMemoryTime = s.AggrMemoryTime
   103  
   104  		// initialize the AggrCount as 0 here since aggr is not started
   105  		stmt.AggrCount = 0
   106  		return stmt
   107  	}
   108  	return nil
   109  }
   110  
   111  func StatementInfoUpdate(ctx context.Context, existing, new Item) {
   112  
   113  	e := existing.(*StatementInfo)
   114  	n := new.(*StatementInfo)
   115  	// nil aggregated stmt record's txn-id, if including diff transactions.
   116  	if e.TransactionID != n.TransactionID {
   117  		e.TransactionID = NilTxnID
   118  	}
   119  	if e.AggrCount == 0 {
   120  		// initialize the AggrCount as 1 here since aggr is started
   121  		windowSize, _ := ctx.Value(DurationKey).(time.Duration)
   122  		e.StatementTag = ""
   123  		e.StatementFingerprint = ""
   124  		e.Error = nil
   125  		e.Database = ""
   126  		duration := e.Duration
   127  		e.AggrMemoryTime = mustDecimal128(convertFloat64ToDecimal128(e.statsArray.GetMemorySize() * float64(duration)))
   128  		e.RequestAt = e.ResponseAt.Truncate(windowSize)
   129  		e.ResponseAt = e.RequestAt.Add(windowSize)
   130  		e.AggrCount = 1
   131  	}
   132  	// update the stats
   133  	if GetTracerProvider().enableStmtMerge {
   134  		e.StmtBuilder.WriteString(";\n")
   135  		e.StmtBuilder.WriteString(n.Statement)
   136  	}
   137  	e.AggrCount += 1
   138  	e.Duration += n.Duration
   139  	e.ResultCount += n.ResultCount
   140  	// responseAt is the last response time
   141  	n.ExecPlan2Stats(context.Background())
   142  	if err := mergeStats(e, n); err != nil {
   143  		// handle error
   144  		logutil.Error("Failed to merge stats", logutil.ErrorField(err))
   145  	}
   146  	n.FreeExecPlan()
   147  	// NO need n.mux.Lock()
   148  	// Because of this op is between EndStatement and FillRow.
   149  	// This function is called in Aggregator, and StatementInfoFilter must return true.
   150  	n.exported = true
   151  }
   152  
   153  func StatementInfoFilter(i Item) bool {
   154  	// Attempt to perform a type assertion to *StatementInfo
   155  	statementInfo, ok := i.(*StatementInfo)
   156  
   157  	if !ok {
   158  		// The item couldn't be cast to *StatementInfo
   159  		return false
   160  	}
   161  
   162  	// Do not aggr the running statement
   163  	if statementInfo.Status == StatementStatusRunning {
   164  		return false
   165  	}
   166  
   167  	// for #14926
   168  	if statementInfo.statsArray.GetCU() < 0 {
   169  		return false
   170  	}
   171  
   172  	// Check SqlSourceType
   173  	switch statementInfo.SqlSourceType {
   174  	case constant.InternalSql, constant.ExternSql, constant.CloudNoUserSql:
   175  		// Check StatementType
   176  		switch statementInfo.StatementType {
   177  		case "Insert", "Update", "Delete", "Execute", "Select":
   178  			if statementInfo.Duration <= GetTracerProvider().selectAggrThreshold {
   179  				return true
   180  			}
   181  		}
   182  	}
   183  	// If no conditions matched, return false
   184  	return false
   185  }
   186  
   187  type StatementInfo struct {
   188  	StatementID          [16]byte `json:"statement_id"`
   189  	TransactionID        [16]byte `json:"transaction_id"`
   190  	SessionID            [16]byte `jons:"session_id"`
   191  	Account              string   `json:"account"`
   192  	User                 string   `json:"user"`
   193  	Host                 string   `json:"host"`
   194  	RoleId               uint32   `json:"role_id"`
   195  	Database             string   `json:"database"`
   196  	Statement            string   `json:"statement"`
   197  	StmtBuilder          strings.Builder
   198  	StatementFingerprint string    `json:"statement_fingerprint"`
   199  	StatementTag         string    `json:"statement_tag"`
   200  	SqlSourceType        string    `json:"sql_source_type"`
   201  	RequestAt            time.Time `json:"request_at"` // see WithRequestAt
   202  
   203  	StatementType string `json:"statement_type"`
   204  	QueryType     string `json:"query_type"`
   205  
   206  	// after
   207  	Status     StatementInfoStatus `json:"status"`
   208  	Error      error               `json:"error"`
   209  	ResponseAt time.Time           `json:"response_at"`
   210  	Duration   time.Duration       `json:"duration"` // unit: ns
   211  	// new ExecPlan
   212  	ExecPlan SerializableExecPlan `json:"-"` // set by SetSerializableExecPlan
   213  	// RowsRead, BytesScan generated from ExecPlan
   214  	RowsRead  int64 `json:"rows_read"`  // see ExecPlan2Json
   215  	BytesScan int64 `json:"bytes_scan"` // see ExecPlan2Json
   216  	AggrCount int64 `json:"aggr_count"` // see EndStatement
   217  
   218  	// AggrMemoryTime
   219  	AggrMemoryTime types.Decimal128
   220  
   221  	ResultCount int64 `json:"result_count"` // see EndStatement
   222  
   223  	ConnType statistic.ConnType `json:"-"` // see frontend.RecordStatement
   224  
   225  	// flow ctrl
   226  	// #		|case 1 |case 2 |case 3 |case 4|
   227  	// end		| false | false | true  | true |  (set true at EndStatement)
   228  	// exported	| false | true  | false | true |  (set true at function FillRow, set false at function EndStatement)
   229  	//
   230  	// case 1: first gen statement_info record
   231  	// case 2: statement_info exported as `status=Running` record
   232  	// case 3: while query done, call EndStatement mark statement need to be exported again
   233  	// case 4: done final export
   234  	//
   235  	// normally    flow: case 1->2->3->4
   236  	// query-quick flow: case 1->3->4
   237  	end bool // cooperate with mux
   238  	mux sync.Mutex
   239  	// reported mark reported
   240  	// set by ReportStatement
   241  	reported bool
   242  	// exported mark exported
   243  	// set by FillRow or StatementInfoUpdate
   244  	exported bool
   245  
   246  	// keep []byte as elem
   247  	jsonByte   []byte
   248  	statsArray statistic.StatsArray
   249  	stated     bool
   250  
   251  	// skipTxnOnce, readonly, for flow control
   252  	// see more on NeedSkipTxn() and SkipTxnId()
   253  	skipTxnOnce bool
   254  	skipTxnID   []byte
   255  }
   256  
   257  type Key struct {
   258  	SessionID     [16]byte
   259  	StatementType string
   260  	Window        time.Time
   261  	Status        StatementInfoStatus
   262  	SqlSourceType string
   263  }
   264  
   265  var stmtPool = sync.Pool{
   266  	New: func() any {
   267  		return &StatementInfo{}
   268  	},
   269  }
   270  
   271  func NewStatementInfo() *StatementInfo {
   272  	s := stmtPool.Get().(*StatementInfo)
   273  	s.statsArray.Reset()
   274  	s.stated = false
   275  	return s
   276  }
   277  
   278  type Statistic struct {
   279  	RowsRead  int64
   280  	BytesScan int64
   281  }
   282  
   283  func (s *StatementInfo) Key(duration time.Duration) interface{} {
   284  	return Key{SessionID: s.SessionID, StatementType: s.StatementType, Window: s.ResponseAt.Truncate(duration), Status: s.Status, SqlSourceType: s.SqlSourceType}
   285  }
   286  
   287  func (s *StatementInfo) GetName() string {
   288  	return SingleStatementTable.GetName()
   289  }
   290  
   291  func (s *StatementInfo) IsMoLogger() bool {
   292  	return s.Account == "sys" && s.User == db_holder.MOLoggerUser
   293  }
   294  
   295  // deltaContentLength approximate value that may gen as table record
   296  // stmtID, txnID, sesID: 36 * 3
   297  // timestamp: 26 * 2
   298  // status: 7
   299  // spanInfo: 36+16
   300  const deltaStmtContentLength = int64(36*3 + 26*2 + 7 + 36 + 16)
   301  const jsonByteLength = int64(4096)
   302  
   303  func (s *StatementInfo) Size() int64 {
   304  	num := int64(unsafe.Sizeof(s)) + deltaStmtContentLength + int64(
   305  		len(s.Account)+len(s.User)+len(s.Host)+
   306  			len(s.Database)+len(s.Statement)+len(s.StatementFingerprint)+len(s.StatementTag)+
   307  			len(s.SqlSourceType)+len(s.StatementType)+len(s.QueryType)+len(s.jsonByte)+len(s.statsArray)*8,
   308  	)
   309  	if s.jsonByte == nil {
   310  		return num + jsonByteLength
   311  	}
   312  	return num
   313  }
   314  
   315  // FreeExecPlan will free StatementInfo.ExecPlan.
   316  // Please make sure it called after StatementInfo.ExecPlan2Stats
   317  func (s *StatementInfo) FreeExecPlan() {
   318  	if s.ExecPlan != nil {
   319  		s.ExecPlan.Free()
   320  		s.ExecPlan = nil
   321  	}
   322  }
   323  
   324  func (s *StatementInfo) Free() {
   325  	s.mux.Lock()
   326  	defer s.mux.Unlock()
   327  	if s.end && s.exported { // cooperate with s.mux
   328  		s.free()
   329  	}
   330  }
   331  
   332  // freeNoLocked will free StatementInfo if StatementInfo.end is true.
   333  // Please make sure it called after EndStatement.
   334  func (s *StatementInfo) freeNoLocked() {
   335  	if s.end {
   336  		s.free()
   337  	}
   338  }
   339  
   340  func (s *StatementInfo) free() {
   341  	s.StatementID = NilStmtID
   342  	s.TransactionID = NilTxnID
   343  	s.SessionID = NilSesID
   344  	s.Account = ""
   345  	s.User = ""
   346  	s.Host = ""
   347  	s.RoleId = 0
   348  	s.Database = ""
   349  	s.Statement = ""
   350  	s.StmtBuilder.Reset()
   351  	s.StatementFingerprint = ""
   352  	s.StatementTag = ""
   353  	s.SqlSourceType = ""
   354  	s.RequestAt = time.Time{}
   355  	s.StatementType = ""
   356  	s.QueryType = ""
   357  	s.Status = StatementStatusRunning
   358  	s.Error = nil
   359  	s.ResponseAt = time.Time{}
   360  	s.Duration = 0
   361  	s.FreeExecPlan() // handle s.ExecPlan
   362  	s.RowsRead = 0
   363  	s.BytesScan = 0
   364  	s.AggrCount = 0
   365  	// s.AggrMemoryTime // skip
   366  	s.ResultCount = 0
   367  	s.ConnType = 0
   368  	s.end = false
   369  	s.reported = false
   370  	s.exported = false
   371  	// clean []byte
   372  	s.jsonByte = nil
   373  	s.statsArray.Reset()
   374  	s.stated = false
   375  	// clean skipTxn ctrl
   376  	s.skipTxnOnce = false
   377  	s.skipTxnID = nil
   378  	stmtPool.Put(s)
   379  }
   380  
   381  func (s *StatementInfo) GetTable() *table.Table { return SingleStatementTable }
   382  
   383  func (s *StatementInfo) FillRow(ctx context.Context, row *table.Row) {
   384  	s.mux.Lock()
   385  	defer s.mux.Unlock()
   386  	s.exported = true
   387  	row.Reset()
   388  	row.SetColumnVal(stmtIDCol, table.UuidField(s.StatementID[:]))
   389  	if !s.IsZeroTxnID() {
   390  		row.SetColumnVal(txnIDCol, table.UuidField(s.TransactionID[:]))
   391  	}
   392  	row.SetColumnVal(sesIDCol, table.UuidField(s.SessionID[:]))
   393  	row.SetColumnVal(accountCol, table.StringField(s.Account))
   394  	row.SetColumnVal(roleIdCol, table.Int64Field(int64(s.RoleId)))
   395  	row.SetColumnVal(userCol, table.StringField(s.User))
   396  	row.SetColumnVal(hostCol, table.StringField(s.Host))
   397  	row.SetColumnVal(dbCol, table.StringField(s.Database))
   398  	row.SetColumnVal(stmtCol, table.StringField(s.Statement))
   399  	row.SetColumnVal(stmtTagCol, table.StringField(s.StatementTag))
   400  	row.SetColumnVal(sqlTypeCol, table.StringField(s.SqlSourceType))
   401  	row.SetColumnVal(stmtFgCol, table.StringField(s.StatementFingerprint))
   402  	row.SetColumnVal(nodeUUIDCol, table.StringField(GetNodeResource().NodeUuid))
   403  	row.SetColumnVal(nodeTypeCol, table.StringField(GetNodeResource().NodeType))
   404  	row.SetColumnVal(reqAtCol, table.TimeField(s.RequestAt))
   405  	row.SetColumnVal(respAtCol, table.TimeField(s.ResponseAt))
   406  	row.SetColumnVal(durationCol, table.Uint64Field(uint64(s.Duration)))
   407  	row.SetColumnVal(statusCol, table.StringField(s.Status.String()))
   408  	if s.Error != nil {
   409  		var moError *moerr.Error
   410  		errCode := moerr.ErrInfo
   411  		if errors.As(s.Error, &moError) {
   412  			errCode = moError.ErrorCode()
   413  		}
   414  		row.SetColumnVal(errCodeCol, table.StringField(fmt.Sprintf("%d", errCode)))
   415  		row.SetColumnVal(errorCol, table.StringField(fmt.Sprintf("%s", s.Error)))
   416  	}
   417  	execPlan := s.ExecPlan2Json(ctx)
   418  	if s.AggrCount > 0 {
   419  		float64Val := calculateAggrMemoryBytes(s.AggrMemoryTime, float64(s.Duration))
   420  		s.statsArray.WithMemorySize(float64Val)
   421  	}
   422  	// stats := s.ExecPlan2Stats(ctx) // deprecated
   423  	stats := s.GetStatsArrayBytes()
   424  	if GetTracerProvider().disableSqlWriter {
   425  		// Be careful, this two string is unsafe, will be free after Free
   426  		row.SetColumnVal(execPlanCol, table.StringField(util.UnsafeBytesToString(execPlan)))
   427  		row.SetColumnVal(statsCol, table.StringField(util.UnsafeBytesToString(stats)))
   428  	} else {
   429  		row.SetColumnVal(execPlanCol, table.BytesField(execPlan))
   430  		row.SetColumnVal(statsCol, table.BytesField(stats))
   431  	}
   432  	row.SetColumnVal(rowsReadCol, table.Int64Field(s.RowsRead))
   433  	row.SetColumnVal(bytesScanCol, table.Int64Field(s.BytesScan))
   434  	row.SetColumnVal(stmtTypeCol, table.StringField(s.StatementType))
   435  	row.SetColumnVal(queryTypeCol, table.StringField(s.QueryType))
   436  	row.SetColumnVal(aggrCntCol, table.Int64Field(s.AggrCount))
   437  	row.SetColumnVal(resultCntCol, table.Int64Field(s.ResultCount))
   438  }
   439  
   440  // calculateAggrMemoryBytes return scale = statistic.Decimal128ToFloat64Scale float64 val
   441  func calculateAggrMemoryBytes(dividend types.Decimal128, divisor float64) float64 {
   442  	scale := int32(statistic.Decimal128ToFloat64Scale)
   443  	divisorD := mustDecimal128(types.Decimal128FromFloat64(divisor, Decimal128Width, scale))
   444  	val, valScale, err := dividend.Div(divisorD, 0, scale)
   445  	val = mustDecimal128(val, err)
   446  	return types.Decimal128ToFloat64(val, valScale)
   447  }
   448  
   449  // mergeStats n (new one) into e (existing one)
   450  // All data generated in ExecPlan2Stats should be handled here, including:
   451  // - statsArray
   452  // - RowRead
   453  // - BytesScan
   454  func mergeStats(e, n *StatementInfo) error {
   455  	e.statsArray.Add(&n.statsArray)
   456  	val, _, err := e.AggrMemoryTime.Add(
   457  		mustDecimal128(convertFloat64ToDecimal128(n.statsArray.GetMemorySize()*float64(n.Duration))),
   458  		Decimal128Scale,
   459  		Decimal128Scale,
   460  	)
   461  	e.AggrMemoryTime = mustDecimal128(val, err)
   462  	e.RowsRead += n.RowsRead
   463  	e.BytesScan += n.BytesScan
   464  	return nil
   465  }
   466  
   467  var noExecPlan = []byte(`{}`)
   468  
   469  // ExecPlan2Json return ExecPlan Serialized json-str //
   470  // please used in s.mux.Lock()
   471  func (s *StatementInfo) ExecPlan2Json(ctx context.Context) []byte {
   472  	if s.jsonByte != nil {
   473  		goto endL
   474  	} else if s.ExecPlan == nil {
   475  		return noExecPlan
   476  	} else {
   477  		s.jsonByte = s.ExecPlan.Marshal(ctx)
   478  		//if queryTime := GetTracerProvider().longQueryTime; queryTime > int64(s.Duration) {
   479  		//	// get nil ExecPlan json-str
   480  		//	jsonByte, _, _ = s.SerializeExecPlan(ctx, nil, uuid.UUID(s.StatementID))
   481  		//}
   482  	}
   483  endL:
   484  	return s.jsonByte
   485  }
   486  
   487  // ExecPlan2Stats return Stats Serialized int array str
   488  // and set RowsRead, BytesScan from ExecPlan
   489  // and CalculateCU
   490  func (s *StatementInfo) ExecPlan2Stats(ctx context.Context) error {
   491  	var stats Statistic
   492  	var statsArray statistic.StatsArray
   493  
   494  	if s.ExecPlan != nil && !s.stated {
   495  		statsArray, stats = s.ExecPlan.Stats(ctx)
   496  		if s.statsArray.GetTimeConsumed() > 0 {
   497  			logutil.GetSkip1Logger().Error("statsArray.GetTimeConsumed() > 0",
   498  				zap.String("statement_id", uuid.UUID(s.StatementID).String()),
   499  			)
   500  		}
   501  		s.statsArray.InitIfEmpty().Add(&statsArray)
   502  		s.statsArray.WithConnType(s.ConnType)
   503  		s.RowsRead = stats.RowsRead
   504  		s.BytesScan = stats.BytesScan
   505  		s.stated = true
   506  	}
   507  	cu := CalculateCU(s.statsArray, int64(s.Duration))
   508  	s.statsArray.WithCU(cu)
   509  	return nil
   510  }
   511  
   512  func (s *StatementInfo) GetStatsArrayBytes() []byte {
   513  	return s.statsArray.ToJsonString()
   514  }
   515  
   516  // SetSkipTxn set skip txn flag, cooperate with SetSkipTxnId()
   517  // usage:
   518  // Step1: SetSkipTxn(true)
   519  // Step2:
   520  //
   521  //	if NeedSkipTxn() {
   522  //		SetSkipTxn(false)
   523  //		SetSkipTxnId(target_txn_id)
   524  //	} else SkipTxnId(current_txn_id) {
   525  //		// record current txn id
   526  //	}
   527  func (s *StatementInfo) SetSkipTxn(skip bool)   { s.skipTxnOnce = skip }
   528  func (s *StatementInfo) SetSkipTxnId(id []byte) { s.skipTxnID = id }
   529  func (s *StatementInfo) NeedSkipTxn() bool      { return s.skipTxnOnce }
   530  func (s *StatementInfo) SkipTxnId(id []byte) bool {
   531  	// s.skipTxnID == nil, means NO skipTxnId
   532  	return s.skipTxnID != nil && bytes.Equal(s.skipTxnID, id)
   533  }
   534  
   535  func GetLongQueryTime() time.Duration {
   536  	return time.Duration(GetTracerProvider().longQueryTime)
   537  }
   538  
   539  type SerializeExecPlanFunc func(ctx context.Context, plan any, uuid2 uuid.UUID) (jsonByte []byte, statsJson statistic.StatsArray, stats Statistic)
   540  
   541  type SerializableExecPlan interface {
   542  	Marshal(context.Context) []byte
   543  	Free()
   544  	Stats(ctx context.Context) (statistic.StatsArray, Statistic)
   545  }
   546  
   547  func (s *StatementInfo) SetSerializableExecPlan(execPlan SerializableExecPlan) {
   548  	s.mux.Lock()
   549  	defer s.mux.Unlock()
   550  	s.ExecPlan = execPlan
   551  }
   552  
   553  func (s *StatementInfo) SetTxnID(id []byte) {
   554  	copy(s.TransactionID[:], id)
   555  }
   556  
   557  func (s *StatementInfo) IsZeroTxnID() bool {
   558  	return bytes.Equal(s.TransactionID[:], NilTxnID[:])
   559  }
   560  
   561  func (s *StatementInfo) Report(ctx context.Context) {
   562  	ReportStatement(ctx, s)
   563  }
   564  
   565  func (s *StatementInfo) MarkResponseAt() {
   566  	if s.ResponseAt.IsZero() {
   567  		s.ResponseAt = time.Now()
   568  		s.Duration = s.ResponseAt.Sub(s.RequestAt)
   569  	}
   570  }
   571  
   572  // TcpIpv4HeaderSize default tcp header bytes.
   573  const TcpIpv4HeaderSize = 66
   574  
   575  // ResponseErrPacketSize avg prefix size for mysql packet response error.
   576  // 66: default tcp header bytes.
   577  // 13: avg payload prefix of err response
   578  const ResponseErrPacketSize = TcpIpv4HeaderSize + 13
   579  
   580  func EndStatement(ctx context.Context, err error, sentRows int64, outBytes int64, outPacket int64) {
   581  	if !GetTracerProvider().IsEnable() {
   582  		return
   583  	}
   584  	s := StatementFromContext(ctx)
   585  	if s == nil {
   586  		panic(moerr.NewInternalError(ctx, "no statement info in context"))
   587  	}
   588  	s.mux.Lock()
   589  	defer s.mux.Unlock()
   590  	if !s.end { // cooperate with s.mux
   591  		// do report
   592  		s.end = true
   593  		s.ResultCount = sentRows
   594  		s.AggrCount = 0
   595  		s.MarkResponseAt()
   596  		// --- Start of metric part
   597  		// duration is filled in s.MarkResponseAt()
   598  		incStatementCounter(s.Account, s.QueryType)
   599  		addStatementDurationCounter(s.Account, s.QueryType, s.Duration)
   600  		// --- END of metric part
   601  		if err != nil {
   602  			outBytes += ResponseErrPacketSize + int64(len(err.Error()))
   603  		}
   604  		if GetTracerProvider().tcpPacket {
   605  			outBytes += TcpIpv4HeaderSize * outPacket
   606  		}
   607  		s.statsArray.InitIfEmpty().WithOutTrafficBytes(float64(outBytes)).WithOutPacketCount(float64(outPacket))
   608  		s.ExecPlan2Stats(ctx)
   609  		if s.statsArray.GetCU() < 0 {
   610  			logutil.Warnf("negative cu: %f, %s", s.statsArray.GetCU(), uuid.UUID(s.StatementID).String())
   611  			v2.GetTraceNegativeCUCounter("cu").Inc()
   612  		} else {
   613  			metric.StatementCUCounter(s.Account, s.SqlSourceType).Add(s.statsArray.GetCU())
   614  		}
   615  		s.Status = StatementStatusSuccess
   616  		if err != nil {
   617  			s.Error = err
   618  			s.Status = StatementStatusFailed
   619  		}
   620  		if !s.reported || s.exported { // cooperate with s.mux
   621  			s.exported = false
   622  			s.Report(ctx)
   623  		}
   624  	}
   625  }
   626  
   627  func addStatementDurationCounter(tenant, queryType string, duration time.Duration) {
   628  	metric.StatementDuration(tenant, queryType).Add(float64(duration))
   629  }
   630  func incStatementCounter(tenant, queryType string) {
   631  	metric.StatementCounter(tenant, queryType).Inc()
   632  }
   633  
   634  type StatementInfoStatus int
   635  
   636  const (
   637  	StatementStatusRunning StatementInfoStatus = iota
   638  	StatementStatusSuccess
   639  	StatementStatusFailed
   640  )
   641  
   642  func (s StatementInfoStatus) String() string {
   643  	switch s {
   644  	case StatementStatusSuccess:
   645  		return "Success"
   646  	case StatementStatusRunning:
   647  		return "Running"
   648  	case StatementStatusFailed:
   649  		return "Failed"
   650  	}
   651  	return "Unknown"
   652  }
   653  
   654  type StatementOption interface {
   655  	Apply(*StatementInfo)
   656  }
   657  
   658  type StatementOptionFunc func(*StatementInfo)
   659  
   660  var ReportStatement = func(ctx context.Context, s *StatementInfo) error {
   661  	if !GetTracerProvider().IsEnable() {
   662  		return nil
   663  	}
   664  	// Filter out the Running record.
   665  	if s.Status == StatementStatusRunning && GetTracerProvider().skipRunningStmt {
   666  		return nil
   667  	}
   668  	// Filter out the MO_LOGGER SQL statements
   669  	//if s.User == db_holder.MOLoggerUser {
   670  	//	goto DiscardAndFreeL
   671  	//}
   672  
   673  	// Filter out the statement is empty
   674  	if s.Statement == "" {
   675  		goto DiscardAndFreeL
   676  	}
   677  
   678  	// Filter out exported or reported statement
   679  	if s.exported || s.reported {
   680  		goto DiscardAndFreeL
   681  	}
   682  
   683  	// Filter out part of the internal SQL statements
   684  	// Todo: review how to aggregate the internal SQL statements logging
   685  	if s.User == "internal" && s.Account == "sys" {
   686  		if s.StatementType == "Commit" || s.StatementType == "Start Transaction" || s.StatementType == "Use" {
   687  			goto DiscardAndFreeL
   688  		}
   689  	}
   690  
   691  	// logging the statement that should not be here anymore
   692  	if s.exported || s.reported || s.Statement == "" {
   693  		logutil.Error("StatementInfo should not be here anymore", zap.String("StatementInfo", s.Statement), zap.String("statement_id", uuid.UUID(s.StatementID).String()), zap.String("user", s.User), zap.Bool("exported", s.exported), zap.Bool("reported", s.reported))
   694  	}
   695  
   696  	s.reported = true
   697  	return GetGlobalBatchProcessor().Collect(ctx, s)
   698  DiscardAndFreeL:
   699  	s.freeNoLocked()
   700  	return nil
   701  }