github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/structs/evaluationstructs.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package structs
    18  
    19  import (
    20  	"bytes"
    21  	"fmt"
    22  	"math"
    23  	"net"
    24  	"net/url"
    25  	"regexp"
    26  	"sort"
    27  	"strconv"
    28  	"strings"
    29  	"sync"
    30  	"time"
    31  
    32  	"github.com/axiomhq/hyperloglog"
    33  	"github.com/dustin/go-humanize"
    34  
    35  	"github.com/siglens/siglens/pkg/segment/utils"
    36  )
    37  
    38  // These structs are used to organize boolean, string, and numeric expressions.
    39  // The expressions can contain fields. To evaluate one of these expressions,
    40  // use GetFields() to find all the fields used directly or indirectly by the
    41  // expression, then create a fieldToValue map specifying the value for each of
    42  // these fields, and call Evaluate(fieldToValue).
    43  type BoolExpr struct {
    44  	IsTerminal bool
    45  
    46  	// Only used when IsTerminal is true.
    47  	LeftValue  *ValueExpr
    48  	RightValue *ValueExpr
    49  	ValueOp    string       // Only = or != for strings; can also be <, <=, >, >= for numbers.
    50  	ValueList  []*ValueExpr //Use for in(<value>, <list>)
    51  
    52  	// Only used when IsTerminal is false. For a unary BoolOp, RightExpr should be nil.
    53  	LeftBool  *BoolExpr
    54  	RightBool *BoolExpr
    55  	BoolOp    BoolOperator
    56  }
    57  
    58  type RenameExpr struct {
    59  	RenameExprMode  RenameExprMode
    60  	OriginalPattern string
    61  	NewPattern      string
    62  }
    63  
    64  type RexExpr struct {
    65  	Pattern     string
    66  	FieldName   string
    67  	RexColNames []string
    68  }
    69  
    70  type StatisticExpr struct {
    71  	StatisticFunctionMode StatisticFunctionMode
    72  	Limit                 string
    73  	StatisticOptions      *StatisticOptions
    74  	FieldList             []string //Must have FieldList
    75  	ByClause              []string
    76  }
    77  
    78  type StatisticOptions struct {
    79  	CountField   string
    80  	OtherStr     string
    81  	PercentField string
    82  	ShowCount    bool
    83  	ShowPerc     bool
    84  	UseOther     bool
    85  }
    86  
    87  type DedupExpr struct {
    88  	Limit              uint64
    89  	FieldList          []string // Must have FieldList
    90  	DedupOptions       *DedupOptions
    91  	DedupSortEles      []*SortElement
    92  	DedupSortAscending []int // Derived from DedupSortEles.SortByAsc values.
    93  
    94  	// DedupCombinations maps combinations to a map mapping the record index
    95  	// (of all included records for this combination) to the sort values for
    96  	// that record. For example, if Limit is 3, each inner map will have at
    97  	// most 3 entries and will store the index and sort values of the top 3
    98  	// records for that combination.
    99  	DedupCombinations map[string]map[int][]SortValue
   100  	PrevCombination   string
   101  
   102  	DedupRecords          map[string]map[string]interface{}
   103  	NumProcessedSegments  uint64
   104  	ProcessedSegmentsLock sync.Mutex
   105  }
   106  
   107  type DedupOptions struct {
   108  	Consecutive bool
   109  	KeepEmpty   bool
   110  	KeepEvents  bool
   111  }
   112  
   113  type SortExpr struct {
   114  	SortEles []*SortElement
   115  	// Limit    int // to be finished
   116  
   117  	SortAscending         []int
   118  	SortRecords           map[string]map[string]interface{}
   119  	NumProcessedSegments  uint64
   120  	ProcessedSegmentsLock sync.Mutex
   121  }
   122  
   123  type SortElement struct {
   124  	SortByAsc bool
   125  	Op        string
   126  	Field     string
   127  }
   128  
   129  type SortValue struct {
   130  	Val         string
   131  	InterpretAs string // Should be "ip", "num", "str", "auto", or ""
   132  }
   133  
   134  // See ValueExprMode type definition for which fields are valid for each mode.
   135  type ValueExpr struct {
   136  	ValueExprMode ValueExprMode
   137  
   138  	FloatValue    float64
   139  	NumericExpr   *NumericExpr
   140  	StringExpr    *StringExpr
   141  	ConditionExpr *ConditionExpr
   142  	BooleanExpr   *BoolExpr
   143  }
   144  
   145  type ConcatExpr struct {
   146  	Atoms []*ConcatAtom
   147  }
   148  
   149  type ConcatAtom struct {
   150  	IsField  bool
   151  	Value    string
   152  	TextExpr *TextExpr
   153  }
   154  
   155  type NumericExpr struct {
   156  	NumericExprMode NumericExprMode
   157  
   158  	IsTerminal bool
   159  
   160  	// Only used when IsTerminal is true.
   161  	ValueIsField bool
   162  	Value        string
   163  
   164  	// Only used when IsTerminal is false.
   165  	Op    string // Either +, -, /, *, abs, ceil, round, sqrt, len
   166  	Left  *NumericExpr
   167  	Right *NumericExpr
   168  	Val   *StringExpr
   169  }
   170  
   171  type StringExpr struct {
   172  	StringExprMode StringExprMode
   173  	RawString      string      // only used when mode is RawString
   174  	FieldName      string      // only used when mode is Field
   175  	ConcatExpr     *ConcatExpr // only used when mode is Concat
   176  	TextExpr       *TextExpr   // only used when mode is TextExpr
   177  }
   178  
   179  type TextExpr struct {
   180  	IsTerminal   bool
   181  	Op           string //lower, ltrim, rtrim
   182  	Value        *StringExpr
   183  	StrToRemove  string
   184  	Delimiter    *StringExpr
   185  	MaxMinValues []*StringExpr
   186  	StartIndex   *NumericExpr
   187  	LengthExpr   *NumericExpr
   188  	Val          *ValueExpr
   189  	Format       *StringExpr
   190  }
   191  
   192  type ConditionExpr struct {
   193  	Op         string //if
   194  	BoolExpr   *BoolExpr
   195  	TrueValue  *ValueExpr //if bool expr is true, take this value
   196  	FalseValue *ValueExpr
   197  }
   198  
   199  type TimechartExpr struct {
   200  	TcOptions  *TcOptions
   201  	BinOptions *BinOptions
   202  	SingleAgg  *SingleAgg
   203  	ByField    string // group by this field inside each time range bucket (timechart)
   204  	LimitExpr  *LimitExpr
   205  }
   206  
   207  type LimitExpr struct {
   208  	IsTop          bool // True: keeps the N highest scoring distinct values of the split-by field
   209  	Num            int
   210  	LimitScoreMode LimitScoreMode
   211  }
   212  
   213  type SingleAgg struct {
   214  	MeasureOperations []*MeasureAggregator
   215  	//Split By clause
   216  }
   217  
   218  type TcOptions struct {
   219  	BinOptions *BinOptions
   220  	UseNull    bool
   221  	UseOther   bool
   222  	NullStr    string
   223  	OtherStr   string
   224  }
   225  
   226  type BinOptions struct {
   227  	SpanOptions *SpanOptions
   228  }
   229  
   230  type SpanOptions struct {
   231  	DefaultSettings bool
   232  	SpanLength      *SpanLength
   233  }
   234  
   235  type SpanLength struct {
   236  	Num       int
   237  	TimeScalr utils.TimeUnit
   238  }
   239  
   240  type SplitByClause struct {
   241  	Field     string
   242  	TcOptions *TcOptions
   243  	// Where clause: to be finished
   244  }
   245  
   246  // This structure is used to store values which are not within limit. And These values will be merged into the 'other' category.
   247  type TMLimitResult struct {
   248  	ValIsInLimit     map[string]bool
   249  	GroupValScoreMap map[string]*utils.CValueEnclosure
   250  	Hll              *hyperloglog.Sketch
   251  	StrSet           map[string]struct{}
   252  	OtherCValArr     []*utils.CValueEnclosure
   253  }
   254  
   255  type BoolOperator uint8
   256  
   257  const (
   258  	BoolOpNot BoolOperator = iota
   259  	BoolOpAnd
   260  	BoolOpOr
   261  )
   262  
   263  type StatisticFunctionMode uint8
   264  
   265  const (
   266  	SFMTop = iota
   267  	SFMRare
   268  )
   269  
   270  type RenameExprMode uint8
   271  
   272  const (
   273  	REMPhrase   = iota //Rename with a phrase
   274  	REMRegex           //Rename fields with similar names using a wildcard
   275  	REMOverride        //Rename to a existing field
   276  )
   277  
   278  type LimitScoreMode uint8
   279  
   280  const (
   281  	LSMBySum  = iota // If only a single aggregation is specified, the score is based on the sum of the values in the aggregation
   282  	LSMByFreq        // Otherwise the score is based on the frequency of the by field's val
   283  )
   284  
   285  type ValueExprMode uint8
   286  
   287  const (
   288  	VEMNumericExpr   = iota // Only NumricExpr is valid
   289  	VEMStringExpr           // Only StringExpr is valid
   290  	VEMConditionExpr        // Only ConditionExpr is valud
   291  	VEMBooleanExpr          // Only BooleanExpr is valid
   292  )
   293  
   294  type StringExprMode uint8
   295  
   296  const (
   297  	SEMRawString  = iota // only used when mode is RawString
   298  	SEMField             // only used when mode is Field
   299  	SEMConcatExpr        // only used when mode is Concat
   300  	SEMTextExpr          // only used when mode is TextExpr
   301  )
   302  
   303  type NumericExprMode uint8
   304  
   305  const (
   306  	NEMNumber      = iota // only used when mode is a Number
   307  	NEMLenString          // only used when mode is a str (used for len())
   308  	NEMNumberField        // only used when mode is Field (Field can be evaluated to a float)
   309  	NEMLenField           // only used when mode is Field (Field can not be evaluated to a float, used for len())
   310  	NEMNumericExpr        // only used when mode is a NumericExpr
   311  )
   312  
   313  // Evaluate this BoolExpr to a boolean, replacing each field in the expression
   314  // with the value specified by fieldToValue. Each field listed by GetFields()
   315  // must be in fieldToValue.
   316  func (self *BoolExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (bool, error) {
   317  	if self.IsTerminal {
   318  		if self.ValueOp == "in" {
   319  			inFlag, err := isInValueList(fieldToValue, self.LeftValue, self.ValueList)
   320  			if err != nil {
   321  				return false, fmt.Errorf("BoolExpr.Evaluate: can not evaluate Eval In function: %v", err)
   322  			}
   323  			return inFlag, err
   324  		} else if self.ValueOp == "isbool" {
   325  			val, err := self.LeftValue.EvaluateToString(fieldToValue)
   326  			if err != nil {
   327  				return false, fmt.Errorf("isbool: can not evaluate to String: %v", err)
   328  			}
   329  			isBool := strings.ToLower(val) == "true" || strings.ToLower(val) == "false" || val == "0" || val == "1"
   330  			return isBool, nil
   331  
   332  		} else if self.ValueOp == "isint" {
   333  			val, err := self.LeftValue.EvaluateToString(fieldToValue)
   334  			if err != nil {
   335  				return false, err
   336  			}
   337  
   338  			_, parseErr := strconv.Atoi(val)
   339  			return parseErr == nil, nil
   340  		} else if self.ValueOp == "isstr" {
   341  			_, floatErr := self.LeftValue.EvaluateToFloat(fieldToValue)
   342  
   343  			if floatErr == nil {
   344  				return false, nil
   345  			}
   346  
   347  			_, strErr := self.LeftValue.EvaluateToString(fieldToValue)
   348  			return strErr == nil, nil
   349  		} else if self.ValueOp == "isnull" {
   350  			// Get the fields associated with this expression
   351  			fields := self.GetFields()
   352  			if len(fields) == 0 {
   353  				return false, fmt.Errorf("BoolExpr.Evaluate: No fields found for isnull operation")
   354  			}
   355  
   356  			// Check the first field's value in the fieldToValue map
   357  			value, exists := fieldToValue[fields[0]]
   358  			if !exists {
   359  				return false, fmt.Errorf("BoolExpr.Evaluate: Field '%s' not found in data", fields[0])
   360  			}
   361  			// Check if the value's Dtype is SS_DT_BACKFILL
   362  			if value.Dtype == utils.SS_DT_BACKFILL {
   363  				return true, nil
   364  			}
   365  			return false, nil
   366  		} else if self.ValueOp == "like" {
   367  			leftStr, errLeftStr := self.LeftValue.EvaluateToString(fieldToValue)
   368  			if errLeftStr != nil {
   369  				return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating left side of LIKE to string: %v", errLeftStr)
   370  			}
   371  
   372  			rightStr, errRightStr := self.RightValue.EvaluateToString(fieldToValue)
   373  			if errRightStr != nil {
   374  				return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating right side of LIKE to string: %v", errRightStr)
   375  			}
   376  
   377  			regexPattern := strings.Replace(strings.Replace(regexp.QuoteMeta(rightStr), "%", ".*", -1), "_", ".", -1)
   378  			matched, err := regexp.MatchString("^"+regexPattern+"$", leftStr)
   379  			if err != nil {
   380  				return false, fmt.Errorf("BoolExpr.Evaluate: regex error in LIKE operation: %v", err)
   381  			}
   382  			return matched, nil
   383  		} else if self.ValueOp == "match" {
   384  			leftStr, errLeftStr := self.LeftValue.EvaluateToString(fieldToValue)
   385  			if errLeftStr != nil {
   386  				return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating left side of MATCH to string: %v", errLeftStr)
   387  			}
   388  
   389  			rightStr, errRightStr := self.RightValue.EvaluateToString(fieldToValue)
   390  			if errRightStr != nil {
   391  				return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating right side of MATCH to string: %v", errRightStr)
   392  			}
   393  
   394  			matched, err := regexp.MatchString(rightStr, leftStr)
   395  			if err != nil {
   396  				return false, fmt.Errorf("BoolExpr.Evaluate: regex error in MATCH operation: %v", err)
   397  			}
   398  			return matched, nil
   399  
   400  		} else if self.ValueOp == "cidrmatch" {
   401  			cidrStr, errCidr := self.LeftValue.EvaluateToString(fieldToValue)
   402  			ipStr, errIp := self.RightValue.EvaluateToString(fieldToValue)
   403  			if errCidr != nil || errIp != nil {
   404  				return false, fmt.Errorf("cidrmatch: error evaluating arguments: %v, %v", errCidr, errIp)
   405  			}
   406  
   407  			match, err := isIPInCIDR(cidrStr, ipStr)
   408  			if err != nil {
   409  				return false, fmt.Errorf("cidrmatch: error in matching CIDR: %v", err)
   410  			}
   411  			return match, nil
   412  		}
   413  
   414  		leftStr, errLeftStr := self.LeftValue.EvaluateToString(fieldToValue)
   415  		rightStr, errRightStr := self.RightValue.EvaluateToString(fieldToValue)
   416  		leftFloat, errLeftFloat := self.LeftValue.EvaluateToFloat(fieldToValue)
   417  		rightFloat, errRightFloat := self.RightValue.EvaluateToFloat(fieldToValue)
   418  
   419  		if errLeftFloat == nil && errRightFloat == nil {
   420  			switch self.ValueOp {
   421  			case "=":
   422  				return leftFloat == rightFloat, nil
   423  			case "!=":
   424  				return leftFloat != rightFloat, nil
   425  			case "<":
   426  				return leftFloat < rightFloat, nil
   427  			case ">":
   428  				return leftFloat > rightFloat, nil
   429  			case "<=":
   430  				return leftFloat <= rightFloat, nil
   431  			case ">=":
   432  				return leftFloat >= rightFloat, nil
   433  			default:
   434  				return false, fmt.Errorf("BoolExpr.Evaluate: invalid ValueOp %v for floats", self.ValueOp)
   435  			}
   436  		} else if errLeftStr == nil && errRightStr == nil {
   437  			switch self.ValueOp {
   438  			case "=":
   439  				return leftStr == rightStr, nil
   440  			case "!=":
   441  
   442  				return leftStr != rightStr, nil
   443  			default:
   444  				return false, fmt.Errorf("BoolExpr.Evaluate: invalid ValueOp %v for strings", self.ValueOp)
   445  			}
   446  		} else {
   447  			if errLeftStr != nil && errLeftFloat != nil {
   448  				return false, fmt.Errorf("BoolExpr.Evaluate: left cannot be evaluated to a string or float")
   449  			}
   450  			if errRightStr != nil && errRightFloat != nil {
   451  				return false, fmt.Errorf("BoolExpr.Evaluate: right cannot be evaluated to a string or float")
   452  			}
   453  
   454  			return false, fmt.Errorf("BoolExpr.Evaluate: left and right ValueExpr have different types")
   455  		}
   456  	} else { // IsTerminal is false
   457  		left, err := self.LeftBool.Evaluate(fieldToValue)
   458  		if err != nil {
   459  			return false, err
   460  		}
   461  
   462  		var right bool
   463  		if self.RightBool != nil {
   464  			var err error
   465  			right, err = self.RightBool.Evaluate(fieldToValue)
   466  			if err != nil {
   467  				return false, err
   468  			}
   469  		}
   470  
   471  		switch self.BoolOp {
   472  		case BoolOpNot:
   473  			return !left, nil
   474  		case BoolOpAnd:
   475  			return left && right, nil
   476  		case BoolOpOr:
   477  			return left || right, nil
   478  		default:
   479  			return false, fmt.Errorf("invalid BoolOp: %v", self.BoolOp)
   480  		}
   481  	}
   482  }
   483  
   484  func isIPInCIDR(cidrStr, ipStr string) (bool, error) {
   485  	_, cidrNet, err := net.ParseCIDR(cidrStr)
   486  	if err != nil {
   487  		return false, err
   488  	}
   489  	ip := net.ParseIP(ipStr)
   490  	if ip == nil {
   491  		return false, fmt.Errorf("invalid IP address")
   492  	}
   493  
   494  	return cidrNet.Contains(ip), nil
   495  }
   496  
   497  func isInValueList(fieldToValue map[string]utils.CValueEnclosure, value *ValueExpr, valueList []*ValueExpr) (bool, error) {
   498  	valueStr, err := value.EvaluateToString(fieldToValue)
   499  	if err != nil {
   500  		return false, fmt.Errorf("isInValueList: can not evaluate to String: %v", err)
   501  	}
   502  
   503  	for _, atom := range valueList {
   504  		atomStr, err := atom.EvaluateToString(fieldToValue)
   505  		if err != nil {
   506  			return false, fmt.Errorf("isInValueList: can not evaluate to String: %v", err)
   507  		}
   508  
   509  		if atomStr == valueStr {
   510  			return true, nil
   511  		}
   512  	}
   513  
   514  	return false, nil
   515  }
   516  
   517  func (self *BoolExpr) GetFields() []string {
   518  	if self.IsTerminal {
   519  		fields := make([]string, 0)
   520  
   521  		if self.RightValue != nil {
   522  			fields = append(fields, self.LeftValue.GetFields()...)
   523  			fields = append(fields, self.RightValue.GetFields()...)
   524  		} else {
   525  			fields = append(fields, self.LeftValue.GetFields()...)
   526  		}
   527  
   528  		//Append fields from the InExpr value list
   529  		for _, ValueExpr := range self.ValueList {
   530  			fields = append(fields, ValueExpr.GetFields()...)
   531  		}
   532  		return fields
   533  	} else {
   534  		// When IsTerminal is false, LeftBool must not be nil, but RightBool will be
   535  		// nil if BoolOp is a unary operation.
   536  		if self.RightBool == nil {
   537  			return self.LeftBool.GetFields()
   538  		}
   539  
   540  		return append(self.LeftBool.GetFields(), self.RightBool.GetFields()...)
   541  	}
   542  }
   543  
   544  // Try evaluating this ValueExpr to a string value, replacing each field in the
   545  // expression with the value specified by fieldToValue. Each field listed by
   546  // GetFields() must be in fieldToValue.
   547  //
   548  // A ValueExpr can be evaluated to a string or float, so if this fails you may
   549  // want to call ValueExpr.EvaluateToFloat().
   550  func (self *ValueExpr) EvaluateToString(fieldToValue map[string]utils.CValueEnclosure) (string, error) {
   551  	switch self.ValueExprMode {
   552  	case VEMStringExpr:
   553  		str, err := self.StringExpr.Evaluate(fieldToValue)
   554  		if err != nil {
   555  			return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to string %v", err)
   556  		}
   557  		return str, nil
   558  	//In this case, field name will be recognized as part of NumericExpr at first. It it can not be converted to float64, it should be evaluated as a str
   559  	case VEMNumericExpr:
   560  		floatValue, err := self.NumericExpr.Evaluate(fieldToValue)
   561  		if err != nil {
   562  			//Because parsing is successful and it can not evaluate as a float in here,
   563  			//There is one possibility: field name may not be float
   564  			str, err := getValueAsString(fieldToValue, self.NumericExpr.Value)
   565  
   566  			if err == nil {
   567  				return str, nil
   568  			}
   569  
   570  			return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to float64 or string: %v", err)
   571  		}
   572  		return strconv.FormatFloat(floatValue, 'f', -1, 64), nil
   573  	case VEMConditionExpr:
   574  		str, err := self.ConditionExpr.EvaluateCondition(fieldToValue)
   575  		if err != nil {
   576  			return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to string %v", err)
   577  		}
   578  		return str, nil
   579  	case VEMBooleanExpr:
   580  		boolResult, err := self.BooleanExpr.Evaluate(fieldToValue)
   581  		if err != nil {
   582  			return "", err
   583  		}
   584  		return strconv.FormatBool(boolResult), nil
   585  	default:
   586  		return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to string")
   587  	}
   588  }
   589  
   590  func (self *StringExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (string, error) {
   591  	switch self.StringExprMode {
   592  	case SEMRawString:
   593  		return self.RawString, nil
   594  	case SEMField:
   595  		if floatValue, err := getValueAsFloat(fieldToValue, self.FieldName); err == nil {
   596  			return strconv.FormatFloat(floatValue, 'f', -1, 64), nil
   597  		}
   598  
   599  		if str, err := getValueAsString(fieldToValue, self.FieldName); err == nil {
   600  			return str, nil
   601  		}
   602  		return "", fmt.Errorf("StringExpr.Evaluate: cannot evaluate to field")
   603  	case SEMConcatExpr:
   604  		return self.ConcatExpr.Evaluate(fieldToValue)
   605  	case SEMTextExpr:
   606  		return self.TextExpr.EvaluateText(fieldToValue)
   607  	default:
   608  		return "", fmt.Errorf("StringExpr.Evaluate: cannot evaluate to string")
   609  	}
   610  }
   611  
   612  // Try evaluating this ValueExpr to a float value, replacing each field in the
   613  // expression with the value specified by fieldToValue. Each field listed by
   614  // GetFields() must be in fieldToValue.
   615  //
   616  // A ValueExpr can be evaluated to a string or float, so if this fails you may
   617  // want to call ValueExpr.EvaluateToString().
   618  func (self *ValueExpr) EvaluateToFloat(fieldToValue map[string]utils.CValueEnclosure) (float64, error) {
   619  	switch self.ValueExprMode {
   620  	case VEMNumericExpr:
   621  		return self.NumericExpr.Evaluate(fieldToValue)
   622  	default:
   623  		return 0, fmt.Errorf("ValueExpr.EvaluateToFloat: cannot evaluate to float")
   624  	}
   625  }
   626  
   627  func (self *ValueExpr) GetFields() []string {
   628  	switch self.ValueExprMode {
   629  	case VEMNumericExpr:
   630  		return self.NumericExpr.GetFields()
   631  	case VEMStringExpr:
   632  		return self.StringExpr.GetFields()
   633  	case VEMConditionExpr:
   634  		return self.ConditionExpr.GetFields()
   635  	case VEMBooleanExpr:
   636  		return self.BooleanExpr.GetFields()
   637  	default:
   638  		return []string{}
   639  	}
   640  }
   641  
   642  func (self *RexExpr) GetFields() []string {
   643  	var fields []string
   644  	fields = append(fields, self.FieldName)
   645  	return fields
   646  }
   647  
   648  func (self *RenameExpr) GetFields() []string {
   649  	fields := make([]string, 0)
   650  
   651  	switch self.RenameExprMode {
   652  	case REMPhrase:
   653  		fallthrough
   654  	case REMOverride:
   655  		fields = append(fields, self.OriginalPattern)
   656  		return fields
   657  	default:
   658  		return []string{}
   659  	}
   660  }
   661  
   662  func (self *StringExpr) GetFields() []string {
   663  	switch self.StringExprMode {
   664  	case SEMConcatExpr:
   665  		return self.ConcatExpr.GetFields()
   666  	case SEMTextExpr:
   667  		return self.TextExpr.GetFields()
   668  	case SEMField:
   669  		return []string{self.FieldName}
   670  	default:
   671  		return []string{}
   672  	}
   673  }
   674  
   675  // Concatenate all the atoms in this ConcatExpr, replacing all fields with the
   676  // values specified by fieldToValue. Each field listed by GetFields() must be in
   677  // fieldToValue.
   678  func (self *ConcatExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (string, error) {
   679  	result := ""
   680  	for _, atom := range self.Atoms {
   681  		if atom.IsField {
   682  			value, err := getValueAsString(fieldToValue, atom.Value)
   683  			if err != nil {
   684  				return "", err
   685  			}
   686  
   687  			result += value
   688  		} else if atom.TextExpr != nil {
   689  			value, err := atom.TextExpr.EvaluateText(fieldToValue)
   690  			if err != nil {
   691  				return "", err
   692  			}
   693  			result += value
   694  		} else {
   695  			result += atom.Value
   696  		}
   697  	}
   698  
   699  	return result, nil
   700  }
   701  
   702  func (self *ConcatExpr) GetFields() []string {
   703  	fields := make([]string, 0)
   704  	for _, atom := range self.Atoms {
   705  		if atom.IsField {
   706  			fields = append(fields, atom.Value)
   707  		} else if atom.TextExpr != nil {
   708  			textFields := atom.TextExpr.GetFields()
   709  			if len(textFields) > 0 {
   710  				fields = append(fields, textFields...)
   711  			}
   712  		}
   713  	}
   714  
   715  	return fields
   716  }
   717  
   718  func GetBucketKey(BucketKey interface{}, keyIndex int) string {
   719  	switch bucketKey := BucketKey.(type) {
   720  	case []string:
   721  		return bucketKey[keyIndex]
   722  	case string:
   723  		return bucketKey
   724  	default:
   725  		return ""
   726  	}
   727  }
   728  
   729  func (self *StatisticExpr) OverrideGroupByCol(bucketResult *BucketResult, resTotal uint64) error {
   730  
   731  	cellValueStr := ""
   732  	for keyIndex, groupByCol := range bucketResult.GroupByKeys {
   733  		if !self.StatisticOptions.ShowCount || !self.StatisticOptions.ShowPerc || (self.StatisticOptions.CountField != groupByCol && self.StatisticOptions.PercentField != groupByCol) {
   734  			continue
   735  		}
   736  
   737  		if self.StatisticOptions.ShowCount && self.StatisticOptions.CountField == groupByCol {
   738  			cellValueStr = strconv.FormatUint(bucketResult.ElemCount, 10)
   739  		}
   740  
   741  		if self.StatisticOptions.ShowPerc && self.StatisticOptions.PercentField == groupByCol {
   742  			percent := float64(bucketResult.ElemCount) / float64(resTotal) * 100
   743  			cellValueStr = fmt.Sprintf("%.6f", percent)
   744  		}
   745  
   746  		// Set the appropriate element of BucketKey to cellValueStr.
   747  		switch bucketKey := bucketResult.BucketKey.(type) {
   748  		case []string:
   749  			bucketKey[keyIndex] = cellValueStr
   750  			bucketResult.BucketKey = bucketKey
   751  		case string:
   752  			if keyIndex != 0 {
   753  				return fmt.Errorf("OverrideGroupByCol: expected keyIndex to be 0, not %v", keyIndex)
   754  			}
   755  			bucketResult.BucketKey = cellValueStr
   756  		default:
   757  			return fmt.Errorf("OverrideGroupByCol: bucket key has unexpected type: %T", bucketKey)
   758  		}
   759  	}
   760  	return nil
   761  }
   762  
   763  func (self *StatisticExpr) SetCountToStatRes(statRes map[string]utils.CValueEnclosure, elemCount uint64) {
   764  	statRes[self.StatisticOptions.CountField] = utils.CValueEnclosure{
   765  		Dtype: utils.SS_DT_UNSIGNED_NUM,
   766  		CVal:  elemCount,
   767  	}
   768  }
   769  
   770  func (self *StatisticExpr) SetPercToStatRes(statRes map[string]utils.CValueEnclosure, elemCount uint64, resTotal uint64) {
   771  	percent := float64(elemCount) / float64(resTotal) * 100
   772  	statRes[self.StatisticOptions.PercentField] = utils.CValueEnclosure{
   773  		Dtype: utils.SS_DT_STRING,
   774  		CVal:  fmt.Sprintf("%.6f", percent),
   775  	}
   776  }
   777  
   778  func (self *StatisticExpr) sortByBucketKey(a, b *BucketResult, fieldList []string, fieldToGroupByKeyIndex map[string]int) bool {
   779  
   780  	for _, field := range fieldList {
   781  		keyIndex := fieldToGroupByKeyIndex[field]
   782  		if GetBucketKey(a.BucketKey, keyIndex) < GetBucketKey(b.BucketKey, keyIndex) {
   783  			return false
   784  		} else if GetBucketKey(a.BucketKey, keyIndex) > GetBucketKey(b.BucketKey, keyIndex) {
   785  			return true
   786  		}
   787  	}
   788  	return true
   789  }
   790  
   791  func (self *StatisticExpr) SortBucketResult(results *[]*BucketResult) error {
   792  
   793  	//GroupByKeys -> BucketKey
   794  	fieldToGroupByKeyIndex := make(map[string]int, len(self.FieldList))
   795  
   796  	//If use the limit option, only the last limit lexicographical of the <field-list> is returned in the search results
   797  	//Therefore, we should sort the bucket by lexicographical value and retain a limited number of values
   798  	if len(self.Limit) > 0 {
   799  
   800  		for index, groupByKey := range (*results)[0].GroupByKeys {
   801  			for _, field := range self.FieldList {
   802  				if field == groupByKey {
   803  					fieldToGroupByKeyIndex[field] = index
   804  				}
   805  			}
   806  		}
   807  
   808  		//Moving the if statement outside of the sorting process can reduce the number of conditional checks
   809  		//Sort results based on the lexicographical value of their field list
   810  		switch self.StatisticFunctionMode {
   811  		case SFMTop:
   812  			sort.Slice(*results, func(index1, index2 int) bool {
   813  				return self.sortByBucketKey((*results)[index1], (*results)[index2], self.FieldList, fieldToGroupByKeyIndex)
   814  			})
   815  		case SFMRare:
   816  			sort.Slice(*results, func(index1, index2 int) bool {
   817  				return !self.sortByBucketKey((*results)[index1], (*results)[index2], self.FieldList, fieldToGroupByKeyIndex)
   818  			})
   819  		}
   820  
   821  		limit, err := strconv.Atoi(self.Limit)
   822  		if err != nil {
   823  			return fmt.Errorf("SortBucketResult: cannot convert %v to int", self.Limit)
   824  		}
   825  
   826  		// Only return unique limit field combinations
   827  		// Since the results are already in order, and there is no Set in Go, we can use a string arr to record the previous field combinations
   828  		// If the current field combination is different from the previous one, it means we have finished processing data for one field combination (we need to process limit combinations in total, limit is a number)
   829  		uniqueFieldsCombination := make([]string, len(self.FieldList))
   830  		combinationCount := 0
   831  		newResults := make([]*BucketResult, 0)
   832  		for _, bucketResult := range *results {
   833  			combinationExist := true
   834  			for index, fieldName := range self.FieldList {
   835  				keyIndex := fieldToGroupByKeyIndex[fieldName]
   836  				val := GetBucketKey(bucketResult.BucketKey, keyIndex)
   837  				if uniqueFieldsCombination[index] != val {
   838  					uniqueFieldsCombination[index] = val
   839  					combinationExist = false
   840  				}
   841  
   842  				statEnclosure, exists := bucketResult.StatRes[fieldName]
   843  				statVal, err := statEnclosure.GetString()
   844  				if exists && err == nil && uniqueFieldsCombination[index] != statVal {
   845  					uniqueFieldsCombination[index] = val
   846  					combinationExist = false
   847  				}
   848  			}
   849  
   850  			// If there is a stats groupby block before statistic groupby block. E.g. ... | stats count BY http_status, gender | rare 1 http_status,
   851  			// In this case, each http_status will be divided by two genders, so we should merge them into one row here
   852  			//Fields combination does not exist
   853  			if !combinationExist {
   854  				combinationCount++
   855  				if combinationCount > limit {
   856  					*results = newResults
   857  					return nil
   858  				}
   859  				newResults = append(newResults, bucketResult)
   860  			} else {
   861  				newResults[combinationCount-1].ElemCount += bucketResult.ElemCount
   862  			}
   863  		}
   864  
   865  	} else { //No limit option, sort results by its values frequency
   866  		switch self.StatisticFunctionMode {
   867  		case SFMTop:
   868  			sort.Slice(*results, func(index1, index2 int) bool {
   869  				return (*results)[index1].ElemCount > (*results)[index2].ElemCount
   870  			})
   871  		case SFMRare:
   872  			sort.Slice(*results, func(index1, index2 int) bool {
   873  				return (*results)[index1].ElemCount < (*results)[index2].ElemCount
   874  			})
   875  		}
   876  	}
   877  
   878  	return nil
   879  }
   880  
   881  // Only display fields which in StatisticExpr
   882  func (self *StatisticExpr) RemoveFieldsNotInExprForBucketRes(bucketResult *BucketResult) error {
   883  	groupByCols := self.GetGroupByCols()
   884  	groupByKeys := make([]string, 0)
   885  	bucketKey := make([]string, 0)
   886  	switch bucketResult.BucketKey.(type) {
   887  	case []string:
   888  		bucketKeyStrs := bucketResult.BucketKey.([]string)
   889  
   890  		for _, field := range groupByCols {
   891  			for rowIndex, groupByCol := range bucketResult.GroupByKeys {
   892  				if field == groupByCol {
   893  					groupByKeys = append(groupByKeys, field)
   894  					bucketKey = append(bucketKey, bucketKeyStrs[rowIndex])
   895  					break
   896  				}
   897  				//Can not find field in GroupByCol, so it may in the StatRes
   898  				val, exists := bucketResult.StatRes[field]
   899  				if exists {
   900  					valStr, _ := val.GetString()
   901  					groupByKeys = append(groupByKeys, field)
   902  					bucketKey = append(bucketKey, valStr)
   903  					delete(bucketResult.StatRes, field)
   904  				}
   905  			}
   906  		}
   907  		bucketResult.BucketKey = bucketKey
   908  	case string:
   909  		if len(groupByCols) == 0 {
   910  			bucketResult.BucketKey = nil
   911  		} else {
   912  			groupByKeys = groupByCols
   913  			// The GroupByCols of the Statistic block increase, so the new columns must come from the Stats function
   914  			if len(groupByCols) > 1 {
   915  				newBucketKey := make([]string, 0)
   916  				for i := 0; i < len(groupByCols); i++ {
   917  					val, exists := bucketResult.StatRes[groupByCols[i]]
   918  					if exists {
   919  						str, err := val.GetString()
   920  						if err != nil {
   921  							return fmt.Errorf("RemoveFieldsNotInExpr: %v", err)
   922  						}
   923  						newBucketKey = append(newBucketKey, str)
   924  					} else {
   925  						newBucketKey = append(newBucketKey, bucketResult.BucketKey.(string))
   926  					}
   927  				}
   928  				bucketResult.BucketKey = newBucketKey
   929  			}
   930  		}
   931  	default:
   932  		return fmt.Errorf("RemoveFieldsNotInExpr: bucket key has unexpected type: %T", bucketKey)
   933  	}
   934  
   935  	// Remove unused func in stats res
   936  	for statColName := range bucketResult.StatRes {
   937  		statColInGroupByCols := false
   938  		for _, groupByCol := range groupByCols {
   939  			if groupByCol == statColName {
   940  				statColInGroupByCols = true
   941  				break
   942  			}
   943  		}
   944  		if !statColInGroupByCols {
   945  			delete(bucketResult.StatRes, statColName)
   946  		}
   947  	}
   948  
   949  	bucketResult.GroupByKeys = groupByKeys
   950  	return nil
   951  }
   952  
   953  func (self *StatisticExpr) GetGroupByCols() []string {
   954  	return append(self.FieldList, self.ByClause...)
   955  }
   956  
   957  func (self *RenameExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure, fieldName string) (string, error) {
   958  	return getValueAsString(fieldToValue, fieldName)
   959  }
   960  
   961  func (self *RexExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure, rexExp *regexp.Regexp) (map[string]string, error) {
   962  
   963  	fieldValue, err := getValueAsString(fieldToValue, self.FieldName)
   964  	if err != nil {
   965  		return nil, fmt.Errorf("RexExpr.Evaluate: %v", err)
   966  	}
   967  
   968  	return MatchAndExtractGroups(fieldValue, rexExp)
   969  }
   970  
   971  func MatchAndExtractGroups(str string, rexExp *regexp.Regexp) (map[string]string, error) {
   972  	match := rexExp.FindStringSubmatch(str)
   973  	if len(match) == 0 {
   974  		return nil, fmt.Errorf("MatchAndExtractGroups: no str in field match the pattern")
   975  	}
   976  	if len(rexExp.SubexpNames()) == 0 {
   977  		return nil, fmt.Errorf("MatchAndExtractGroups: no field create from the pattern")
   978  	}
   979  
   980  	result := make(map[string]string)
   981  	for i, name := range rexExp.SubexpNames() {
   982  		if i != 0 && name != "" {
   983  			result[name] = match[i]
   984  		}
   985  	}
   986  
   987  	return result, nil
   988  }
   989  
   990  // Check if colName match the OriginalPattern
   991  func (self *RenameExpr) CheckIfMatch(colName string) bool {
   992  	regexPattern := `\b` + strings.ReplaceAll(self.OriginalPattern, "*", "(.*)") + `\b`
   993  	regex, err := regexp.Compile(regexPattern)
   994  	if err != nil {
   995  		return false
   996  	}
   997  
   998  	matchingParts := regex.FindStringSubmatch(colName)
   999  	return len(matchingParts) != 0
  1000  }
  1001  
  1002  // Check if colName matches the specified pattern and replace wildcards to generate a new colName.
  1003  func (self *RenameExpr) ProcessRenameRegexExpression(colName string) (string, error) {
  1004  
  1005  	originalPattern := self.OriginalPattern
  1006  	newPattern := self.NewPattern
  1007  
  1008  	regexPattern := `\b` + strings.ReplaceAll(originalPattern, "*", "(.*)") + `\b`
  1009  	regex, err := regexp.Compile(regexPattern)
  1010  	if err != nil {
  1011  		return "", fmt.Errorf("ProcessRenameRegexExpression: There are some errors in the pattern: %v", err)
  1012  	}
  1013  
  1014  	matchingParts := regex.FindStringSubmatch(colName)
  1015  	if len(matchingParts) == 0 {
  1016  		return "", nil
  1017  	}
  1018  
  1019  	result := newPattern
  1020  	for _, match := range matchingParts[1:] {
  1021  		result = strings.Replace(result, "*", match, 1)
  1022  	}
  1023  
  1024  	return result, nil
  1025  }
  1026  
  1027  func (self *RenameExpr) RemoveColsByIndex(strs []string, indexToRemove []int) []string {
  1028  	results := make([]string, 0)
  1029  
  1030  	for index, val := range strs {
  1031  		shouldRemove := false
  1032  		for _, delIndex := range indexToRemove {
  1033  			if delIndex == index {
  1034  				shouldRemove = true
  1035  				break
  1036  			}
  1037  		}
  1038  		if shouldRemove {
  1039  			continue
  1040  		}
  1041  		results = append(results, val)
  1042  	}
  1043  	return results
  1044  }
  1045  
  1046  func (self *RenameExpr) RemoveBucketResGroupByColumnsByIndex(bucketResult *BucketResult, indexToRemove []int) {
  1047  
  1048  	if len(indexToRemove) == 0 {
  1049  		return
  1050  	}
  1051  
  1052  	bucketResult.GroupByKeys = self.RemoveColsByIndex(bucketResult.GroupByKeys, indexToRemove)
  1053  
  1054  	switch bucketKey := bucketResult.BucketKey.(type) {
  1055  	case []string:
  1056  		bucketResult.BucketKey = self.RemoveColsByIndex(bucketKey, indexToRemove)
  1057  	case string:
  1058  		bucketResult.BucketKey = nil
  1059  	}
  1060  
  1061  }
  1062  
  1063  // Remove unused GroupByVals in Bucket Holder
  1064  func (self *RenameExpr) RemoveBucketHolderGroupByColumnsByIndex(bucketHolder *BucketHolder, groupByCols []string, indexToRemove []int) {
  1065  
  1066  	if len(indexToRemove) == 0 {
  1067  		return
  1068  	}
  1069  
  1070  	groupByVals := make([]string, 0)
  1071  	for index := range groupByCols {
  1072  		shouldRemove := false
  1073  		for _, delIndex := range indexToRemove {
  1074  			if delIndex == index {
  1075  				shouldRemove = true
  1076  				break
  1077  			}
  1078  		}
  1079  		if shouldRemove {
  1080  			continue
  1081  		}
  1082  		groupByVals = append(groupByVals, bucketHolder.GroupByValues[index])
  1083  	}
  1084  
  1085  	bucketHolder.GroupByValues = groupByVals
  1086  
  1087  }
  1088  
  1089  // Evaluate this NumericExpr to a float, replacing each field in the expression
  1090  // with the value specified by fieldToValue. Each field listed by GetFields()
  1091  // must be in fieldToValue.
  1092  func (self *NumericExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (float64, error) {
  1093  	if self.Op == "now" {
  1094  		timestamp := time.Now().Unix()
  1095  		return float64(timestamp), nil
  1096  	}
  1097  	if self.IsTerminal {
  1098  		if self.ValueIsField {
  1099  			switch self.NumericExprMode {
  1100  			case NEMNumberField:
  1101  				return getValueAsFloat(fieldToValue, self.Value)
  1102  			case NEMLenField:
  1103  				return float64(len(fieldToValue[self.Value].CVal.(string))), nil
  1104  			}
  1105  		} else {
  1106  			switch self.NumericExprMode {
  1107  			case NEMNumber:
  1108  				value, err := strconv.ParseFloat(self.Value, 64)
  1109  				if err != nil {
  1110  					return 0, fmt.Errorf("NumericExpr.Evaluate: cannot convert %v to float", self.Value)
  1111  				}
  1112  				return value, nil
  1113  			case NEMLenString:
  1114  				value := float64(len(self.Value))
  1115  				return value, nil
  1116  			}
  1117  		}
  1118  		return 0, fmt.Errorf("NumericExpr.Evaluate: cannot convert %v to float", self.Value)
  1119  	} else {
  1120  
  1121  		left := float64(0)
  1122  		var err error
  1123  		if self.Left != nil {
  1124  			left, err = self.Left.Evaluate(fieldToValue)
  1125  			if err != nil {
  1126  
  1127  				return 0, err
  1128  			}
  1129  		}
  1130  
  1131  		var right float64
  1132  		if self.Right != nil {
  1133  			right, err = self.Right.Evaluate(fieldToValue)
  1134  			if err != nil {
  1135  				return 0, err
  1136  			}
  1137  		}
  1138  
  1139  		switch self.Op {
  1140  		case "+":
  1141  			return left + right, nil
  1142  		case "-":
  1143  			return left - right, nil
  1144  		case "*":
  1145  			return left * right, nil
  1146  		case "/":
  1147  			return left / right, nil
  1148  		case "abs":
  1149  			return math.Abs(left), nil
  1150  		case "ceil":
  1151  			return math.Ceil(left), nil
  1152  		case "round":
  1153  			if self.Right != nil {
  1154  				return round(left, int(right)), nil
  1155  			} else {
  1156  				return math.Round(left), nil
  1157  			}
  1158  		case "sqrt":
  1159  			if left < 0 {
  1160  				return -1, fmt.Errorf("NumericExpr.Evaluate: Negative values cannot be converted to square roots: %v", left)
  1161  			}
  1162  			return math.Sqrt(left), nil
  1163  		case "len":
  1164  			return left, nil
  1165  		case "exact":
  1166  			result, err := self.Left.Evaluate(fieldToValue)
  1167  			if err != nil {
  1168  				return 0, err
  1169  			}
  1170  			return result, nil
  1171  		case "exp":
  1172  			exp, err := self.Left.Evaluate(fieldToValue)
  1173  			if err != nil {
  1174  				return 0, err
  1175  			}
  1176  			return math.Exp(exp), nil
  1177  		case "tonumber":
  1178  			if self.Val == nil {
  1179  				return 0, fmt.Errorf("NumericExpr.Evaluate: tonumber operation requires a string expression")
  1180  			}
  1181  			strValue, err := self.Val.Evaluate(fieldToValue)
  1182  			if err != nil {
  1183  				return 0, fmt.Errorf("NumericExpr.Evaluate: Error in tonumber operation: %v", err)
  1184  			}
  1185  			base := 10
  1186  			if self.Right != nil {
  1187  				baseValue, err := self.Right.Evaluate(fieldToValue)
  1188  				if err != nil {
  1189  					return 0, err
  1190  				}
  1191  				base = int(baseValue)
  1192  				if base < 2 || base > 36 {
  1193  					return 0, fmt.Errorf("NumericExpr.Evaluate: Invalid base for tonumber: %v", base)
  1194  				}
  1195  			}
  1196  			number, err := strconv.ParseInt(strValue, base, 64)
  1197  			if err != nil {
  1198  				return 0, fmt.Errorf("NumericExpr.Evaluate: cannot convert '%v' to number with base %d", strValue, base)
  1199  			}
  1200  			return float64(number), nil
  1201  
  1202  		default:
  1203  			return 0, fmt.Errorf("NumericExpr.Evaluate: unexpected operation: %v", self.Op)
  1204  		}
  1205  	}
  1206  }
  1207  
  1208  func (self *TextExpr) EvaluateText(fieldToValue map[string]utils.CValueEnclosure) (string, error) {
  1209  	if self.Op == "max" {
  1210  		if len(self.MaxMinValues) == 0 {
  1211  			return "", fmt.Errorf("TextExpr.Evaluate: no values provided for 'max' operation")
  1212  		}
  1213  		maxString := ""
  1214  		for _, expr := range self.MaxMinValues {
  1215  			result, err := expr.Evaluate(fieldToValue)
  1216  			if err != nil {
  1217  				return "", err
  1218  			}
  1219  			if result > maxString {
  1220  				maxString = result
  1221  			}
  1222  		}
  1223  		return maxString, nil
  1224  
  1225  	} else if self.Op == "min" {
  1226  		if len(self.MaxMinValues) == 0 {
  1227  			return "", fmt.Errorf("TextExpr.Evaluate: no values provided for 'min' operation")
  1228  		}
  1229  		minString := ""
  1230  		for _, expr := range self.MaxMinValues {
  1231  			result, err := expr.Evaluate(fieldToValue)
  1232  			if err != nil {
  1233  				return "", err
  1234  			}
  1235  			if minString == "" || result < minString {
  1236  				minString = result
  1237  			}
  1238  		}
  1239  		return minString, nil
  1240  
  1241  	} else if self.Op == "tostring" {
  1242  		valueStr, err := self.Val.EvaluateToString(fieldToValue)
  1243  		if err != nil {
  1244  			return "", fmt.Errorf("TextExpr.Evaluate: failed to evaluate value for 'tostring' operation: %v", err)
  1245  		}
  1246  		if self.Format != nil {
  1247  			formatStr, err := self.Format.Evaluate(fieldToValue)
  1248  			if err != nil {
  1249  				return "", fmt.Errorf("TextExpr.Evaluate: failed to evaluate format for 'tostring' operation: %v", err)
  1250  			}
  1251  			switch formatStr {
  1252  			case "hex":
  1253  				num, convErr := strconv.Atoi(valueStr)
  1254  				if convErr != nil {
  1255  					return "", fmt.Errorf("TextExpr.Evaluate: failed to convert value '%s' to integer for hex formatting: %v", valueStr, convErr)
  1256  				}
  1257  				return fmt.Sprintf("%#x", num), nil
  1258  			case "commas":
  1259  				num, convErr := strconv.ParseFloat(valueStr, 64)
  1260  				if convErr != nil {
  1261  					return "", fmt.Errorf("TextExpr.Evaluate: failed to convert value '%s' to float for comma formatting: %v", valueStr, convErr)
  1262  				}
  1263  				roundedNum := math.Round(num*100) / 100
  1264  				formattedNum := humanize.CommafWithDigits(roundedNum, 2)
  1265  				return formattedNum, nil
  1266  			case "duration":
  1267  				num, convErr := strconv.Atoi(valueStr)
  1268  				if convErr != nil {
  1269  					return "", fmt.Errorf("TextExpr.Evaluate: failed to convert value '%s' to seconds for duration formatting: %v", valueStr, convErr)
  1270  				}
  1271  				hours := num / 3600
  1272  				minutes := (num % 3600) / 60
  1273  				seconds := num % 60
  1274  				return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds), nil
  1275  			default:
  1276  				return "", fmt.Errorf("TextExpr.Evaluate: unsupported format '%s' for tostring operation", formatStr)
  1277  			}
  1278  		} else {
  1279  			return valueStr, nil
  1280  		}
  1281  	}
  1282  	cellValueStr, err := self.Value.Evaluate(fieldToValue)
  1283  	if err != nil {
  1284  		return "", fmt.Errorf("TextExpr.Evaluate: can not evaluate text as a str: %v", err)
  1285  	}
  1286  
  1287  	switch self.Op {
  1288  	case "lower":
  1289  		return strings.ToLower(cellValueStr), nil
  1290  	case "ltrim":
  1291  		return strings.TrimLeft(cellValueStr, self.StrToRemove), nil
  1292  	case "rtrim":
  1293  		return strings.TrimRight(cellValueStr, self.StrToRemove), nil
  1294  	case "urldecode":
  1295  		decodedStr, decodeErr := url.QueryUnescape(cellValueStr)
  1296  		if decodeErr != nil {
  1297  			return "", fmt.Errorf("TextExpr.Evaluate: failed to decode URL: %v", decodeErr)
  1298  		}
  1299  		return decodedStr, nil
  1300  
  1301  	case "split":
  1302  		delimiterStr, err := self.Delimiter.Evaluate(fieldToValue)
  1303  		if err != nil {
  1304  			return "", fmt.Errorf("TextExpr.Evaluate: cannot evaluate delimiter as a string: %v", err)
  1305  		}
  1306  
  1307  		return strings.Join(strings.Split(cellValueStr, delimiterStr), "&nbsp"), nil
  1308  
  1309  	case "substr":
  1310  		baseString, err := self.Value.Evaluate(fieldToValue)
  1311  		if err != nil {
  1312  			return "", err
  1313  		}
  1314  		startIndexFloat, err := self.StartIndex.Evaluate(fieldToValue)
  1315  		if err != nil {
  1316  			return "", err
  1317  		}
  1318  		startIndex := int(startIndexFloat)
  1319  		if startIndex > 0 {
  1320  			startIndex = startIndex - 1
  1321  		}
  1322  		if startIndex < 0 {
  1323  			startIndex = len(baseString) + startIndex
  1324  		}
  1325  		if startIndex < 0 || startIndex >= len(baseString) {
  1326  			return "", fmt.Errorf("substr: start index is out of range")
  1327  		}
  1328  		substrLength := len(baseString) - startIndex
  1329  		if self.LengthExpr != nil {
  1330  			lengthFloat, err := self.LengthExpr.Evaluate(fieldToValue)
  1331  			if err != nil {
  1332  				return "", err
  1333  			}
  1334  			substrLength = int(lengthFloat)
  1335  			if substrLength < 0 || startIndex+substrLength > len(baseString) {
  1336  				return "", fmt.Errorf("substr: length leads to out of range substring")
  1337  			}
  1338  		}
  1339  		endIndex := startIndex + substrLength
  1340  		if endIndex > len(baseString) {
  1341  			endIndex = len(baseString)
  1342  		}
  1343  		return baseString[startIndex:endIndex], nil
  1344  
  1345  	default:
  1346  		return "", fmt.Errorf("TextExpr.Evaluate: unexpected operation: %v", self.Op)
  1347  	}
  1348  }
  1349  
  1350  // In this case, if we can not evaluate numeric expr to a float, we should evaluate it as a str
  1351  func (self *ValueExpr) EvaluateValueExprAsString(fieldToValue map[string]utils.CValueEnclosure) (string, error) {
  1352  	var str string
  1353  	var err error
  1354  	switch self.ValueExprMode {
  1355  	case VEMNumericExpr:
  1356  		floatValue, err := self.EvaluateToFloat(fieldToValue)
  1357  		str = fmt.Sprintf("%v", floatValue)
  1358  		if err != nil {
  1359  			str, err = self.EvaluateToString(fieldToValue)
  1360  			if err != nil {
  1361  				return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate to a ValueExpr: %v", err)
  1362  			}
  1363  		}
  1364  	case VEMStringExpr:
  1365  		str, err = self.EvaluateToString(fieldToValue)
  1366  		if err != nil {
  1367  			return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate to a ValueExpr: %v", err)
  1368  		}
  1369  	}
  1370  	return str, nil
  1371  }
  1372  
  1373  // Field may come from BoolExpr or ValueExpr
  1374  func (self *ConditionExpr) EvaluateCondition(fieldToValue map[string]utils.CValueEnclosure) (string, error) {
  1375  	predicateFlag, err := self.BoolExpr.Evaluate(fieldToValue)
  1376  	if err != nil {
  1377  		return "", fmt.Errorf("ConditionExpr.Evaluate: %v", err)
  1378  	}
  1379  
  1380  	trueValue, err := self.TrueValue.EvaluateValueExprAsString(fieldToValue)
  1381  	if err != nil {
  1382  		return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate trueValue to a ValueExpr: %v", err)
  1383  	}
  1384  	falseValue, err := self.FalseValue.EvaluateValueExprAsString(fieldToValue)
  1385  	if err != nil {
  1386  		return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate falseValue to a ValueExpr: %v", err)
  1387  	}
  1388  
  1389  	switch self.Op {
  1390  	case "if":
  1391  		if predicateFlag {
  1392  			return trueValue, nil
  1393  		} else {
  1394  			return falseValue, nil
  1395  		}
  1396  	default:
  1397  		return "", fmt.Errorf("ConditionExpr.Evaluate: unexpected operation: %v", self.Op)
  1398  	}
  1399  
  1400  }
  1401  
  1402  func (self *TextExpr) GetFields() []string {
  1403  	fields := make([]string, 0)
  1404  	if self.IsTerminal || (self.Op != "max" && self.Op != "min") {
  1405  		if self.Value != nil {
  1406  			fields = append(fields, self.Value.GetFields()...)
  1407  		}
  1408  		if self.Val != nil {
  1409  			fields = append(fields, self.Val.GetFields()...)
  1410  		}
  1411  		if self.Delimiter != nil {
  1412  			fields = append(fields, self.Delimiter.GetFields()...)
  1413  		}
  1414  		if self.StartIndex != nil {
  1415  			fields = append(fields, self.StartIndex.GetFields()...)
  1416  		}
  1417  		if self.LengthExpr != nil {
  1418  			fields = append(fields, self.LengthExpr.GetFields()...)
  1419  		}
  1420  		if self.Format != nil {
  1421  			fields = append(fields, self.Format.GetFields()...)
  1422  		}
  1423  		return fields
  1424  	}
  1425  	for _, expr := range self.MaxMinValues {
  1426  		fields = append(fields, expr.GetFields()...)
  1427  	}
  1428  	return fields
  1429  
  1430  }
  1431  
  1432  // Append all the fields in ConditionExpr
  1433  func (self *ConditionExpr) GetFields() []string {
  1434  	fields := make([]string, 0)
  1435  	fields = append(fields, self.BoolExpr.GetFields()...)
  1436  	fields = append(fields, self.TrueValue.GetFields()...)
  1437  	fields = append(fields, self.FalseValue.GetFields()...)
  1438  	return fields
  1439  }
  1440  
  1441  // Specifying a value and a precision
  1442  func round(number float64, precision int) float64 {
  1443  	scale := math.Pow10(precision)
  1444  	return math.Round(number*scale) / scale
  1445  }
  1446  
  1447  func (self *NumericExpr) GetFields() []string {
  1448  	fields := make([]string, 0)
  1449  	if self.Val != nil {
  1450  		return append(fields, self.Val.GetFields()...)
  1451  	}
  1452  	if self.IsTerminal {
  1453  		if self.Op == "now" {
  1454  			return fields
  1455  		}
  1456  		if self.ValueIsField {
  1457  			return []string{self.Value}
  1458  		} else {
  1459  			return []string{}
  1460  		}
  1461  	} else if self.Right != nil {
  1462  		return append(self.Left.GetFields(), self.Right.GetFields()...)
  1463  	} else {
  1464  		return self.Left.GetFields()
  1465  	}
  1466  }
  1467  
  1468  func getValueAsString(fieldToValue map[string]utils.CValueEnclosure, field string) (string, error) {
  1469  	enclosure, ok := fieldToValue[field]
  1470  	if !ok {
  1471  		return "", fmt.Errorf("Missing field %v", field)
  1472  	}
  1473  
  1474  	return enclosure.GetString()
  1475  }
  1476  
  1477  func getValueAsFloat(fieldToValue map[string]utils.CValueEnclosure, field string) (float64, error) {
  1478  	enclosure, ok := fieldToValue[field]
  1479  	if !ok {
  1480  		return 0, fmt.Errorf("Missing field %v", field)
  1481  	}
  1482  
  1483  	if value, err := enclosure.GetFloatValue(); err == nil {
  1484  		return value, nil
  1485  	}
  1486  
  1487  	// Check if the string value is a number.
  1488  	if enclosure.Dtype == utils.SS_DT_STRING {
  1489  		if value, err := strconv.ParseFloat(enclosure.CVal.(string), 64); err == nil {
  1490  			return value, nil
  1491  		}
  1492  	}
  1493  
  1494  	return 0, fmt.Errorf("Cannot convert CValueEnclosure %v to float", enclosure)
  1495  }
  1496  
  1497  func (self *SortValue) Compare(other *SortValue) (int, error) {
  1498  	switch self.InterpretAs {
  1499  	case "ip":
  1500  		selfIP := net.ParseIP(self.Val)
  1501  		otherIP := net.ParseIP(other.Val)
  1502  		if selfIP == nil || otherIP == nil {
  1503  			return 0, fmt.Errorf("SortValue.Compare: cannot parse IP address")
  1504  		}
  1505  		return bytes.Compare(selfIP, otherIP), nil
  1506  	case "num":
  1507  		selfFloat, selfErr := strconv.ParseFloat(self.Val, 64)
  1508  		otherFloat, otherErr := strconv.ParseFloat(other.Val, 64)
  1509  		if selfErr != nil || otherErr != nil {
  1510  			return 0, fmt.Errorf("SortValue.Compare: cannot parse %v and %v as float", self.Val, other.Val)
  1511  		}
  1512  
  1513  		if selfFloat == otherFloat {
  1514  			return 0, nil
  1515  		} else if selfFloat < otherFloat {
  1516  			return -1, nil
  1517  		} else {
  1518  			return 1, nil
  1519  		}
  1520  	case "str":
  1521  		return strings.Compare(self.Val, other.Val), nil
  1522  	case "auto", "":
  1523  		selfFloat, selfErr := strconv.ParseFloat(self.Val, 64)
  1524  		otherFloat, otherErr := strconv.ParseFloat(other.Val, 64)
  1525  		if selfErr == nil && otherErr == nil {
  1526  			if selfFloat == otherFloat {
  1527  				return 0, nil
  1528  			} else if selfFloat < otherFloat {
  1529  				return -1, nil
  1530  			} else {
  1531  				return 1, nil
  1532  			}
  1533  		}
  1534  
  1535  		selfIp := net.ParseIP(self.Val)
  1536  		otherIp := net.ParseIP(other.Val)
  1537  		if selfIp != nil && otherIp != nil {
  1538  			return bytes.Compare(selfIp, otherIp), nil
  1539  		}
  1540  
  1541  		return strings.Compare(self.Val, other.Val), nil
  1542  	default:
  1543  		return 0, fmt.Errorf("SortValue.Compare: invalid InterpretAs value: %v", self.InterpretAs)
  1544  	}
  1545  }
  1546  
  1547  // The `ascending` slice should have the same length as `a` and `b`. Moreover,
  1548  // each element of `ascending` should be either +1 or -1; +1 means higher
  1549  // values get sorted higher, and -1 means lower values get sorted higher.
  1550  func CompareSortValueSlices(a []SortValue, b []SortValue, ascending []int) (int, error) {
  1551  	if len(a) != len(b) || len(a) != len(ascending) {
  1552  		return 0, fmt.Errorf("CompareSortValueSlices: slices have different lengths")
  1553  	}
  1554  
  1555  	for i := 0; i < len(a); i++ {
  1556  		comp, err := a[i].Compare(&b[i])
  1557  		if err != nil {
  1558  			return 0, fmt.Errorf("CompareSortValueSlices: %v", err)
  1559  		}
  1560  
  1561  		if comp != 0 {
  1562  			return comp * ascending[i], nil
  1563  		}
  1564  	}
  1565  
  1566  	return 0, nil
  1567  }