github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/aggregations/segaggs_test.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package aggregations
    18  
    19  import (
    20  	"encoding/json"
    21  	"fmt"
    22  	"math/rand"
    23  	"strings"
    24  	"testing"
    25  
    26  	"github.com/siglens/siglens/pkg/segment/structs"
    27  	"github.com/siglens/siglens/pkg/segment/utils"
    28  	"github.com/stretchr/testify/assert"
    29  )
    30  
    31  type SimpleSearchExpr struct {
    32  	Op             string
    33  	Field          string
    34  	Values         interface{}
    35  	ValueIsRegex   bool
    36  	ExprType       utils.SS_DTYPE
    37  	DtypeEnclosure *utils.DtypeEnclosure
    38  }
    39  
    40  func Test_conditionMatch(t *testing.T) {
    41  	tests := []struct {
    42  		name         string
    43  		fieldValue   interface{}
    44  		op           string
    45  		searchValue  interface{}
    46  		expectedBool bool
    47  	}{
    48  		{"EqualStringTrue", "test", "=", "test", true},
    49  		{"EqualStringFalse", "test", "=", "fail", false},
    50  		{"NotEqualStringTrue", "test", "!=", "fail", true},
    51  		{"NotEqualStringFalse", "test", "!=", "test", false},
    52  		{"GreaterThanTrue", 10, ">", 5, true},
    53  		{"GreaterThanFalse", 3, ">", 5, false},
    54  		{"GreaterThanOrEqualTrue", 5, ">=", 5, true},
    55  		{"GreaterThanOrEqualFalse", 3, ">=", 5, false},
    56  		{"LessThanTrue", 3, "<", 5, true},
    57  		{"LessThanFalse", 10, "<", 5, false},
    58  		{"LessThanOrEqualTrue", 5, "<=", 5, true},
    59  		{"LessThanOrEqualFalse", 10, "<=", 5, false},
    60  		{"InvalidOperator", 10, "invalid", 5, false},
    61  		{"InvalidFieldValue", "invalid", ">", 5, false},
    62  		{"InvalidSearchValue", 5, ">", "invalid", false},
    63  	}
    64  
    65  	for _, test := range tests {
    66  		t.Run(test.name, func(t *testing.T) {
    67  			result := conditionMatch(test.fieldValue, test.op, test.searchValue)
    68  			assert.Equal(t, test.expectedBool, result)
    69  		})
    70  	}
    71  }
    72  
    73  var cities = []string{
    74  	"Hyderabad",
    75  	"New York",
    76  	"Los Angeles",
    77  	"Chicago",
    78  	"Houston",
    79  	"Phoenix",
    80  	"Philadelphia",
    81  	"San Antonio",
    82  	"San Diego",
    83  	"Dallas",
    84  	"San Jose",
    85  }
    86  
    87  var countries = []string{
    88  	"India",
    89  	"United States",
    90  	"Canada",
    91  	"United Kingdom",
    92  	"Australia",
    93  	"Germany",
    94  	"France",
    95  	"Spain",
    96  	"Italy",
    97  	"Japan",
    98  }
    99  
   100  func generateTestRecords(numRecords int) map[string]map[string]interface{} {
   101  	records := make(map[string]map[string]interface{}, numRecords)
   102  
   103  	for i := 0; i < numRecords; i++ {
   104  		record := make(map[string]interface{})
   105  
   106  		record["timestamp"] = uint64(1659874108987)
   107  		record["city"] = cities[rand.Intn(len(cities))]
   108  		record["gender"] = []string{"male", "female"}[rand.Intn(2)]
   109  		record["country"] = countries[rand.Intn(len(countries))]
   110  		record["http_method"] = []string{"GET", "POST", "PUT", "DELETE"}[rand.Intn(4)]
   111  		record["http_status"] = []int64{200, 201, 301, 302, 404}[rand.Intn(5)]
   112  		record["latitude"] = rand.Float64() * 180
   113  		record["longitude"] = rand.Float64() * 180
   114  
   115  		records[fmt.Sprint(i)] = record
   116  	}
   117  
   118  	return records
   119  }
   120  
   121  func getFinalColsForGeneratedTestRecords() map[string]bool {
   122  	return map[string]bool{"timestamp": true, "city": true, "gender": true, "country": true, "http_method": true, "http_status": true, "latitude": true, "longitude": true}
   123  }
   124  
   125  // Test Cases for processTransactionsOnRecords
   126  func All_TestCasesForTransactionCommands() (map[int]bool, []*structs.TransactionArguments, map[int]map[string]interface{}) {
   127  	matchesSomeRecords := make(map[int]bool)
   128  	searchResults := make(map[int]map[string]interface{})
   129  
   130  	// CASE 1: Only Fields
   131  	txnArgs1 := &structs.TransactionArguments{
   132  		Fields:     []string{"gender", "city"},
   133  		StartsWith: nil,
   134  		EndsWith:   nil,
   135  	}
   136  	matchesSomeRecords[1] = true
   137  
   138  	// CASE 2: Only EndsWith
   139  	txnArgs2 := &structs.TransactionArguments{
   140  		EndsWith:   &structs.FilterStringExpr{StringValue: "DELETE"},
   141  		StartsWith: &structs.FilterStringExpr{StringValue: "GET"},
   142  		Fields:     []string{},
   143  	}
   144  	matchesSomeRecords[2] = true
   145  
   146  	// CASE 3: Only StartsWith
   147  	txnArgs3 := &structs.TransactionArguments{
   148  		StartsWith: &structs.FilterStringExpr{StringValue: "GET"},
   149  		EndsWith:   nil,
   150  		Fields:     []string{},
   151  	}
   152  	matchesSomeRecords[3] = true
   153  
   154  	// CASE 4: StartsWith and EndsWith
   155  	txnArgs4 := &structs.TransactionArguments{
   156  		StartsWith: &structs.FilterStringExpr{StringValue: "GET"},
   157  		EndsWith:   &structs.FilterStringExpr{StringValue: "DELETE"},
   158  		Fields:     []string{},
   159  	}
   160  	matchesSomeRecords[4] = true
   161  
   162  	// CASE 5: StartsWith and EndsWith and one Field
   163  	txnArgs5 := &structs.TransactionArguments{
   164  		StartsWith: &structs.FilterStringExpr{StringValue: "GET"},
   165  		EndsWith:   &structs.FilterStringExpr{StringValue: "DELETE"},
   166  		Fields:     []string{"gender"},
   167  	}
   168  	matchesSomeRecords[5] = true
   169  
   170  	// CASE 6: StartsWith and EndsWith and two Fields
   171  	txnArgs6 := &structs.TransactionArguments{
   172  		StartsWith: &structs.FilterStringExpr{StringValue: "GET"},
   173  		EndsWith:   &structs.FilterStringExpr{StringValue: "DELETE"},
   174  		Fields:     []string{"gender", "country"},
   175  	}
   176  	matchesSomeRecords[6] = true
   177  
   178  	// CASE 7: StartsWith and EndsWith with String Clauses only OR: startswith=("GET" OR "POST1") endswith=("DELETE" OR "POST2")
   179  	txnArgs7 := &structs.TransactionArguments{
   180  		StartsWith: &structs.FilterStringExpr{
   181  			SearchNode: &structs.ASTNode{
   182  				OrFilterCondition: &structs.Condition{
   183  					FilterCriteria: []*structs.FilterCriteria{
   184  						{
   185  							MatchFilter: &structs.MatchFilter{
   186  								MatchColumn: "*",
   187  								MatchWords: [][]byte{
   188  									[]byte("GET"),
   189  								},
   190  								MatchOperator: utils.And,
   191  								MatchPhrase:   []byte("GET"),
   192  								MatchType:     structs.MATCH_PHRASE,
   193  							},
   194  						},
   195  						{
   196  							MatchFilter: &structs.MatchFilter{
   197  								MatchColumn: "*",
   198  								MatchWords: [][]byte{
   199  									[]byte("POST1"),
   200  								},
   201  								MatchOperator: utils.And,
   202  								MatchPhrase:   []byte("POST1"),
   203  								MatchType:     structs.MATCH_PHRASE,
   204  							},
   205  						},
   206  					},
   207  				},
   208  			},
   209  		},
   210  		EndsWith: &structs.FilterStringExpr{
   211  			SearchNode: &structs.ASTNode{
   212  				OrFilterCondition: &structs.Condition{
   213  					FilterCriteria: []*structs.FilterCriteria{
   214  						{
   215  							MatchFilter: &structs.MatchFilter{
   216  								MatchColumn: "*",
   217  								MatchWords: [][]byte{
   218  									[]byte("DELETE"),
   219  								},
   220  								MatchOperator: utils.And,
   221  								MatchPhrase:   []byte("DELETE"),
   222  								MatchType:     structs.MATCH_PHRASE,
   223  							},
   224  						},
   225  						{
   226  							MatchFilter: &structs.MatchFilter{
   227  								MatchColumn: "*",
   228  								MatchWords: [][]byte{
   229  									[]byte("POST2"),
   230  								},
   231  								MatchOperator: utils.And,
   232  								MatchPhrase:   []byte("POST2"),
   233  								MatchType:     structs.MATCH_PHRASE,
   234  							},
   235  						},
   236  					},
   237  				},
   238  			},
   239  		},
   240  		Fields: []string{"gender", "country"},
   241  	}
   242  	matchesSomeRecords[7] = true
   243  	searchResults[7] = map[string]interface{}{
   244  		"startswith": [][]*SimpleSearchExpr{
   245  			{
   246  				{
   247  					Op:           "=",
   248  					Field:        "http_method",
   249  					Values:       "GET",
   250  					ValueIsRegex: false,
   251  					ExprType:     utils.SS_DT_STRING,
   252  					DtypeEnclosure: &utils.DtypeEnclosure{
   253  						Dtype:     utils.SS_DT_STRING,
   254  						StringVal: "GET",
   255  					},
   256  				},
   257  				{
   258  					Op:           "=",
   259  					Field:        "http_method",
   260  					Values:       "POST1",
   261  					ValueIsRegex: false,
   262  					ExprType:     utils.SS_DT_STRING,
   263  					DtypeEnclosure: &utils.DtypeEnclosure{
   264  						Dtype:     utils.SS_DT_STRING,
   265  						StringVal: "POST1",
   266  					},
   267  				},
   268  			},
   269  		},
   270  		"endswith": [][]*SimpleSearchExpr{
   271  			{
   272  				{
   273  					Op:           "=",
   274  					Field:        "http_method",
   275  					Values:       "DELETE",
   276  					ValueIsRegex: false,
   277  					ExprType:     utils.SS_DT_STRING,
   278  					DtypeEnclosure: &utils.DtypeEnclosure{
   279  						Dtype:     utils.SS_DT_STRING,
   280  						StringVal: "DELETE",
   281  					},
   282  				},
   283  				{
   284  					Op:           "=",
   285  					Field:        "http_method",
   286  					Values:       "POST2",
   287  					ValueIsRegex: false,
   288  					ExprType:     utils.SS_DT_STRING,
   289  					DtypeEnclosure: &utils.DtypeEnclosure{
   290  						Dtype:     utils.SS_DT_STRING,
   291  						StringVal: "POST2",
   292  					},
   293  				},
   294  			},
   295  		},
   296  	}
   297  
   298  	// CASE 8: StartsWith and EndsWith with String Clauses only AND (Negative Case): startswith=("GET" AND "POST2") endswith=("POST")
   299  	txnArgs8 := &structs.TransactionArguments{
   300  		StartsWith: &structs.FilterStringExpr{
   301  			SearchNode: &structs.ASTNode{
   302  				AndFilterCondition: &structs.Condition{
   303  					FilterCriteria: []*structs.FilterCriteria{
   304  						{
   305  							MatchFilter: &structs.MatchFilter{
   306  								MatchColumn: "*",
   307  								MatchWords: [][]byte{
   308  									[]byte("GET"),
   309  								},
   310  								MatchOperator: utils.And,
   311  								MatchPhrase:   []byte("GET"),
   312  								MatchType:     structs.MATCH_PHRASE,
   313  							},
   314  						},
   315  						{
   316  							MatchFilter: &structs.MatchFilter{
   317  								MatchColumn: "*",
   318  								MatchWords: [][]byte{
   319  									[]byte("POST2"),
   320  								},
   321  								MatchOperator: utils.And,
   322  								MatchPhrase:   []byte("POST2"),
   323  								MatchType:     structs.MATCH_PHRASE,
   324  							},
   325  						},
   326  					},
   327  				},
   328  			},
   329  		},
   330  		EndsWith: &structs.FilterStringExpr{
   331  			SearchNode: &structs.ASTNode{
   332  				AndFilterCondition: &structs.Condition{
   333  					FilterCriteria: []*structs.FilterCriteria{
   334  						{
   335  							MatchFilter: &structs.MatchFilter{
   336  								MatchColumn: "*",
   337  								MatchWords: [][]byte{
   338  									[]byte("DELETE"),
   339  								},
   340  								MatchOperator: utils.And,
   341  								MatchPhrase:   []byte("DELETE"),
   342  								MatchType:     structs.MATCH_PHRASE,
   343  							},
   344  						},
   345  					},
   346  				},
   347  			},
   348  		},
   349  		Fields: []string{"gender", "country"},
   350  	}
   351  	matchesSomeRecords[8] = false
   352  
   353  	// CASE 9: StartsWith and EndsWith with String Clauses only AND (Positive Case): startswith=("GET" AND "male") endswith=("DELETE")
   354  	txnArgs9 := &structs.TransactionArguments{
   355  		Fields: []string{"gender", "country"},
   356  		StartsWith: &structs.FilterStringExpr{
   357  			SearchNode: &structs.ASTNode{
   358  				AndFilterCondition: &structs.Condition{
   359  					FilterCriteria: []*structs.FilterCriteria{
   360  						{
   361  							MatchFilter: &structs.MatchFilter{
   362  								MatchColumn: "*",
   363  								MatchWords: [][]byte{
   364  									[]byte("GET"),
   365  								},
   366  								MatchOperator: utils.And,
   367  								MatchPhrase:   []byte("GET"),
   368  								MatchType:     structs.MATCH_PHRASE,
   369  							},
   370  						},
   371  						{
   372  							MatchFilter: &structs.MatchFilter{
   373  								MatchColumn: "*",
   374  								MatchWords: [][]byte{
   375  									[]byte("male"),
   376  								},
   377  								MatchOperator: utils.And,
   378  								MatchPhrase:   []byte("male"),
   379  								MatchType:     structs.MATCH_PHRASE,
   380  							},
   381  						},
   382  					},
   383  				},
   384  			},
   385  		},
   386  		EndsWith: &structs.FilterStringExpr{
   387  			SearchNode: &structs.ASTNode{
   388  				AndFilterCondition: &structs.Condition{
   389  					FilterCriteria: []*structs.FilterCriteria{
   390  						{
   391  							MatchFilter: &structs.MatchFilter{
   392  								MatchColumn: "*",
   393  								MatchWords: [][]byte{
   394  									[]byte("DELETE"),
   395  								},
   396  								MatchOperator: utils.And,
   397  								MatchPhrase:   []byte("DELETE"),
   398  								MatchType:     structs.MATCH_PHRASE,
   399  							},
   400  						},
   401  					},
   402  				},
   403  			},
   404  		},
   405  	}
   406  	matchesSomeRecords[9] = true
   407  	searchResults[9] = map[string]interface{}{
   408  		"startswith": [][]*SimpleSearchExpr{
   409  			{
   410  				{
   411  					Op:           "=",
   412  					Field:        "http_method",
   413  					Values:       "GET",
   414  					ValueIsRegex: false,
   415  					ExprType:     utils.SS_DT_STRING,
   416  					DtypeEnclosure: &utils.DtypeEnclosure{
   417  						Dtype:          utils.SS_DT_STRING,
   418  						StringVal:      "GET",
   419  						StringValBytes: []byte("GET"),
   420  					},
   421  				},
   422  			},
   423  			{
   424  				{
   425  					Op:           "=",
   426  					Field:        "gender",
   427  					Values:       "male",
   428  					ValueIsRegex: false,
   429  					ExprType:     utils.SS_DT_STRING,
   430  					DtypeEnclosure: &utils.DtypeEnclosure{
   431  						Dtype:          utils.SS_DT_STRING,
   432  						StringVal:      "male",
   433  						StringValBytes: []byte("male"),
   434  					},
   435  				},
   436  			},
   437  		},
   438  		"endswith": [][]*SimpleSearchExpr{
   439  			{
   440  				{
   441  					Op:           "=",
   442  					Field:        "http_method",
   443  					Values:       "DELETE",
   444  					ValueIsRegex: false,
   445  					ExprType:     utils.SS_DT_STRING,
   446  					DtypeEnclosure: &utils.DtypeEnclosure{
   447  						Dtype:     utils.SS_DT_STRING,
   448  						StringVal: "DELETE",
   449  					},
   450  				},
   451  			},
   452  		},
   453  	}
   454  
   455  	// CASE 10: StartsWith is a Valid Search Expr and EndsWith is String Value: startswith=status>=300 endswith="DELETE"
   456  	txnArgs10 := &structs.TransactionArguments{
   457  		StartsWith: &structs.FilterStringExpr{
   458  			SearchNode: &structs.ASTNode{
   459  				AndFilterCondition: &structs.Condition{
   460  					FilterCriteria: []*structs.FilterCriteria{
   461  						{
   462  							ExpressionFilter: &structs.ExpressionFilter{
   463  								LeftInput: &structs.FilterInput{
   464  									Expression: &structs.Expression{
   465  										LeftInput: &structs.ExpressionInput{
   466  											ColumnValue: nil,
   467  											ColumnName:  "http_status",
   468  										},
   469  										ExpressionOp: utils.Add,
   470  										RightInput:   nil,
   471  									},
   472  								},
   473  								FilterOperator: utils.GreaterThanOrEqualTo,
   474  								RightInput: &structs.FilterInput{
   475  									Expression: &structs.Expression{
   476  										LeftInput: &structs.ExpressionInput{
   477  											ColumnValue: &utils.DtypeEnclosure{
   478  												Dtype:       utils.SS_DT_UNSIGNED_NUM,
   479  												UnsignedVal: uint64(300),
   480  												SignedVal:   int64(300),
   481  												FloatVal:    float64(300),
   482  												StringVal:   "300",
   483  											},
   484  										},
   485  										ExpressionOp: utils.Add,
   486  										RightInput:   nil,
   487  									},
   488  								},
   489  							},
   490  						},
   491  					},
   492  				},
   493  			},
   494  		},
   495  		EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"},
   496  	}
   497  	matchesSomeRecords[10] = true
   498  	searchResults[10] = map[string]interface{}{
   499  		"startswith": [][]*SimpleSearchExpr{
   500  			{
   501  				{
   502  					Op:           ">=",
   503  					Field:        "http_status",
   504  					Values:       json.Number("300"),
   505  					ValueIsRegex: false,
   506  					ExprType:     utils.SS_DT_SIGNED_NUM,
   507  					DtypeEnclosure: &utils.DtypeEnclosure{
   508  						Dtype:       utils.SS_DT_SIGNED_NUM,
   509  						FloatVal:    float64(300),
   510  						UnsignedVal: uint64(300),
   511  						SignedVal:   int64(300),
   512  						StringVal:   "300",
   513  					},
   514  				},
   515  			},
   516  		},
   517  	}
   518  
   519  	// CASE 11: StartsWith is not a Valid Search Term (comparing between two string fields) and EndsWith is String value: startswith=city>"Hyderabad" endswith="DELETE"
   520  	txnArgs11 := &structs.TransactionArguments{
   521  		StartsWith: &structs.FilterStringExpr{
   522  			SearchNode: &structs.ASTNode{
   523  				AndFilterCondition: &structs.Condition{
   524  					FilterCriteria: []*structs.FilterCriteria{
   525  						{
   526  							ExpressionFilter: &structs.ExpressionFilter{
   527  								LeftInput: &structs.FilterInput{
   528  									Expression: &structs.Expression{
   529  										LeftInput: &structs.ExpressionInput{
   530  											ColumnValue: nil,
   531  											ColumnName:  "city",
   532  										},
   533  										ExpressionOp: utils.Add,
   534  										RightInput:   nil,
   535  									},
   536  								},
   537  								FilterOperator: utils.GreaterThan,
   538  								RightInput: &structs.FilterInput{
   539  									Expression: &structs.Expression{
   540  										LeftInput: &structs.ExpressionInput{
   541  											ColumnValue: &utils.DtypeEnclosure{
   542  												Dtype:     utils.SS_DT_STRING,
   543  												StringVal: "Hyderabad",
   544  											},
   545  										},
   546  									},
   547  								},
   548  							},
   549  						},
   550  					},
   551  				},
   552  			},
   553  		},
   554  		EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"},
   555  	}
   556  	matchesSomeRecords[11] = false
   557  
   558  	// CASE 12: StartsWith is not a Valid Search Term (comparing between string and number fields) and EndsWith is String Clause: startswith=city>300 endswith=("DELETE")
   559  	txnArgs12 := &structs.TransactionArguments{
   560  		StartsWith: &structs.FilterStringExpr{
   561  			SearchNode: &structs.ASTNode{
   562  				AndFilterCondition: &structs.Condition{
   563  					FilterCriteria: []*structs.FilterCriteria{
   564  						{
   565  							ExpressionFilter: &structs.ExpressionFilter{
   566  								LeftInput: &structs.FilterInput{
   567  									Expression: &structs.Expression{
   568  										LeftInput: &structs.ExpressionInput{
   569  											ColumnValue: nil,
   570  											ColumnName:  "city",
   571  										},
   572  										ExpressionOp: utils.Add,
   573  										RightInput:   nil,
   574  									},
   575  								},
   576  								FilterOperator: utils.GreaterThan,
   577  								RightInput: &structs.FilterInput{
   578  									Expression: &structs.Expression{
   579  										LeftInput: &structs.ExpressionInput{
   580  											ColumnValue: &utils.DtypeEnclosure{
   581  												Dtype:       utils.SS_DT_UNSIGNED_NUM,
   582  												StringVal:   "300",
   583  												UnsignedVal: uint64(300),
   584  												SignedVal:   int64(300),
   585  												FloatVal:    float64(300),
   586  											},
   587  										},
   588  										ExpressionOp: utils.Add,
   589  										RightInput:   nil,
   590  									},
   591  								},
   592  							},
   593  						},
   594  					},
   595  				},
   596  			},
   597  		},
   598  		EndsWith: &structs.FilterStringExpr{
   599  			SearchNode: &structs.ASTNode{
   600  				AndFilterCondition: &structs.Condition{
   601  					FilterCriteria: []*structs.FilterCriteria{
   602  						{
   603  							MatchFilter: &structs.MatchFilter{
   604  								MatchColumn: "*",
   605  								MatchWords: [][]byte{
   606  									[]byte("DELETE"),
   607  								},
   608  								MatchOperator: utils.And,
   609  								MatchPhrase:   []byte("DELETE"),
   610  								MatchType:     structs.MATCH_PHRASE,
   611  							},
   612  						},
   613  					},
   614  				},
   615  			},
   616  		},
   617  	}
   618  	matchesSomeRecords[12] = false
   619  
   620  	// CASE 13: StartsWith is a Valid Search Term (String1 = String2) and EndsWith is String Value: startswith=city="Hyderabad" endswith="DELETE"
   621  	txnArgs13 := &structs.TransactionArguments{
   622  		StartsWith: &structs.FilterStringExpr{
   623  			SearchNode: &structs.ASTNode{
   624  				AndFilterCondition: &structs.Condition{
   625  					FilterCriteria: []*structs.FilterCriteria{
   626  						{
   627  							ExpressionFilter: &structs.ExpressionFilter{
   628  								LeftInput: &structs.FilterInput{
   629  									Expression: &structs.Expression{
   630  										LeftInput: &structs.ExpressionInput{
   631  											ColumnValue: nil,
   632  											ColumnName:  "city",
   633  										},
   634  										ExpressionOp: utils.Add,
   635  										RightInput:   nil,
   636  									},
   637  								},
   638  								FilterOperator: utils.Equals,
   639  								RightInput: &structs.FilterInput{
   640  									Expression: &structs.Expression{
   641  										LeftInput: &structs.ExpressionInput{
   642  											ColumnValue: &utils.DtypeEnclosure{
   643  												Dtype:     utils.SS_DT_STRING,
   644  												StringVal: "Hyderabad",
   645  											},
   646  										},
   647  									},
   648  								},
   649  							},
   650  						},
   651  					},
   652  				},
   653  			},
   654  		},
   655  		EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"},
   656  	}
   657  	matchesSomeRecords[13] = true
   658  	searchResults[13] = map[string]interface{}{
   659  		"startswith": [][]*SimpleSearchExpr{
   660  			{
   661  				{
   662  					Op:           "=",
   663  					Field:        "city",
   664  					Values:       "Hyderabad",
   665  					ValueIsRegex: false,
   666  					ExprType:     utils.SS_DT_STRING,
   667  					DtypeEnclosure: &utils.DtypeEnclosure{
   668  						Dtype:     utils.SS_DT_STRING,
   669  						StringVal: "Hyderabad",
   670  					},
   671  				},
   672  			},
   673  		},
   674  	}
   675  
   676  	// CASE 14: Eval Expression:  transaction gender startswith=eval(status > 300 AND http_method="POST" OR http_method="PUT")
   677  	txnArgs14 := &structs.TransactionArguments{
   678  		Fields: []string{"gender"},
   679  		StartsWith: &structs.FilterStringExpr{
   680  			EvalBoolExpr: &structs.BoolExpr{
   681  				IsTerminal: false,
   682  				LeftBool: &structs.BoolExpr{
   683  					IsTerminal: true,
   684  					LeftValue: &structs.ValueExpr{
   685  						ValueExprMode: structs.VEMNumericExpr,
   686  						NumericExpr: &structs.NumericExpr{
   687  							NumericExprMode: structs.NEMNumberField,
   688  							IsTerminal:      true,
   689  							ValueIsField:    true,
   690  							Value:           "http_status",
   691  						},
   692  					},
   693  					RightValue: &structs.ValueExpr{
   694  						NumericExpr: &structs.NumericExpr{
   695  							NumericExprMode: structs.NEMNumber,
   696  							IsTerminal:      true,
   697  							ValueIsField:    false,
   698  							Value:           "300",
   699  						},
   700  					},
   701  					ValueOp: ">",
   702  				},
   703  				RightBool: &structs.BoolExpr{
   704  					IsTerminal: false,
   705  					LeftBool: &structs.BoolExpr{
   706  						IsTerminal: true,
   707  						LeftValue: &structs.ValueExpr{
   708  							ValueExprMode: structs.VEMNumericExpr,
   709  							NumericExpr: &structs.NumericExpr{
   710  								NumericExprMode: structs.NEMNumberField,
   711  								IsTerminal:      true,
   712  								ValueIsField:    true,
   713  								Value:           "http_method",
   714  							},
   715  						},
   716  						RightValue: &structs.ValueExpr{
   717  							ValueExprMode: structs.VEMStringExpr,
   718  							StringExpr: &structs.StringExpr{
   719  								StringExprMode: structs.SEMRawString,
   720  								RawString:      "POST",
   721  							},
   722  						},
   723  						ValueOp: "=",
   724  					},
   725  					RightBool: &structs.BoolExpr{
   726  						IsTerminal: true,
   727  						LeftValue: &structs.ValueExpr{
   728  							ValueExprMode: structs.VEMNumericExpr,
   729  							NumericExpr: &structs.NumericExpr{
   730  								NumericExprMode: structs.NEMNumberField,
   731  								IsTerminal:      true,
   732  								ValueIsField:    true,
   733  								Value:           "http_method",
   734  							},
   735  						},
   736  						RightValue: &structs.ValueExpr{
   737  							ValueExprMode: structs.VEMStringExpr,
   738  							StringExpr: &structs.StringExpr{
   739  								StringExprMode: structs.SEMRawString,
   740  								RawString:      "PUT",
   741  							},
   742  						},
   743  						ValueOp: "=",
   744  					},
   745  					BoolOp: structs.BoolOpOr,
   746  				},
   747  				BoolOp: structs.BoolOpAnd,
   748  			},
   749  		},
   750  	}
   751  	matchesSomeRecords[14] = true
   752  	searchResults[14] = map[string]interface{}{
   753  		"startswith": [][]*SimpleSearchExpr{
   754  			{
   755  				{
   756  					Op:           ">",
   757  					Field:        "http_status",
   758  					Values:       json.Number("300"),
   759  					ValueIsRegex: false,
   760  					ExprType:     utils.SS_DT_SIGNED_NUM,
   761  					DtypeEnclosure: &utils.DtypeEnclosure{
   762  						Dtype:       utils.SS_DT_SIGNED_NUM,
   763  						FloatVal:    float64(300),
   764  						UnsignedVal: uint64(300),
   765  						SignedVal:   int64(300),
   766  						StringVal:   "300",
   767  					},
   768  				},
   769  			},
   770  			{
   771  				{
   772  					Op:           "=",
   773  					Field:        "http_method",
   774  					Values:       "POST",
   775  					ValueIsRegex: false,
   776  					ExprType:     utils.SS_DT_STRING,
   777  					DtypeEnclosure: &utils.DtypeEnclosure{
   778  						Dtype:     utils.SS_DT_STRING,
   779  						StringVal: "POST",
   780  					},
   781  				},
   782  				{
   783  					Op:           "=",
   784  					Field:        "http_method",
   785  					Values:       "PUT",
   786  					ValueIsRegex: false,
   787  					ExprType:     utils.SS_DT_STRING,
   788  					DtypeEnclosure: &utils.DtypeEnclosure{
   789  						Dtype:     utils.SS_DT_STRING,
   790  						StringVal: "PUT",
   791  					},
   792  				},
   793  			},
   794  		},
   795  	}
   796  
   797  	// CASE 15: String Search Expr: transaction gender startswith="status>300 OR status=201 AND http_method=POST" endswith=eval(status<400)
   798  	txnArgs15 := &structs.TransactionArguments{
   799  		Fields: []string{"gender"},
   800  		StartsWith: &structs.FilterStringExpr{
   801  			SearchNode: &structs.ASTNode{
   802  				AndFilterCondition: &structs.Condition{
   803  					FilterCriteria: []*structs.FilterCriteria{
   804  						{
   805  							ExpressionFilter: &structs.ExpressionFilter{
   806  								LeftInput: &structs.FilterInput{
   807  									Expression: &structs.Expression{
   808  										LeftInput: &structs.ExpressionInput{
   809  											ColumnValue: nil,
   810  											ColumnName:  "http_method",
   811  										},
   812  										ExpressionOp: utils.Add,
   813  										RightInput:   nil,
   814  									},
   815  								},
   816  								RightInput: &structs.FilterInput{
   817  									Expression: &structs.Expression{
   818  										LeftInput: &structs.ExpressionInput{
   819  											ColumnValue: &utils.DtypeEnclosure{
   820  												Dtype:     utils.SS_DT_STRING,
   821  												StringVal: "POST",
   822  											},
   823  											ColumnName: "",
   824  										},
   825  										ExpressionOp: utils.Add,
   826  										RightInput:   nil,
   827  									},
   828  								},
   829  								FilterOperator: utils.Equals,
   830  							},
   831  						},
   832  					},
   833  					NestedNodes: []*structs.ASTNode{
   834  						{
   835  							OrFilterCondition: &structs.Condition{
   836  								FilterCriteria: []*structs.FilterCriteria{
   837  									{
   838  										ExpressionFilter: &structs.ExpressionFilter{
   839  											LeftInput: &structs.FilterInput{
   840  												Expression: &structs.Expression{
   841  													LeftInput: &structs.ExpressionInput{
   842  														ColumnValue: nil,
   843  														ColumnName:  "http_status",
   844  													},
   845  													ExpressionOp: utils.Add,
   846  													RightInput:   nil,
   847  												},
   848  											},
   849  											RightInput: &structs.FilterInput{
   850  												Expression: &structs.Expression{
   851  													LeftInput: &structs.ExpressionInput{
   852  														ColumnValue: &utils.DtypeEnclosure{
   853  															Dtype:       utils.SS_DT_UNSIGNED_NUM,
   854  															UnsignedVal: uint64(300),
   855  															SignedVal:   int64(300),
   856  															FloatVal:    float64(300),
   857  															StringVal:   "300",
   858  														},
   859  														ColumnName: "",
   860  													},
   861  													ExpressionOp: utils.Add,
   862  													RightInput:   nil,
   863  												},
   864  											},
   865  											FilterOperator: utils.GreaterThan,
   866  										},
   867  									},
   868  									{
   869  										ExpressionFilter: &structs.ExpressionFilter{
   870  											LeftInput: &structs.FilterInput{
   871  												Expression: &structs.Expression{
   872  													LeftInput: &structs.ExpressionInput{
   873  														ColumnValue: nil,
   874  														ColumnName:  "http_status",
   875  													},
   876  													ExpressionOp: utils.Add,
   877  													RightInput:   nil,
   878  												},
   879  											},
   880  											RightInput: &structs.FilterInput{
   881  												Expression: &structs.Expression{
   882  													LeftInput: &structs.ExpressionInput{
   883  														ColumnValue: &utils.DtypeEnclosure{
   884  															Dtype:       utils.SS_DT_UNSIGNED_NUM,
   885  															UnsignedVal: uint64(201),
   886  															SignedVal:   int64(201),
   887  															FloatVal:    float64(201),
   888  															StringVal:   "201",
   889  														},
   890  														ColumnName: "",
   891  													},
   892  													ExpressionOp: utils.Add,
   893  													RightInput:   nil,
   894  												},
   895  											},
   896  											FilterOperator: utils.Equals,
   897  										},
   898  									},
   899  								},
   900  								NestedNodes: nil,
   901  							},
   902  						},
   903  					},
   904  				},
   905  			},
   906  		},
   907  		EndsWith: &structs.FilterStringExpr{
   908  			EvalBoolExpr: &structs.BoolExpr{
   909  				IsTerminal: true,
   910  				LeftValue: &structs.ValueExpr{
   911  					NumericExpr: &structs.NumericExpr{
   912  						IsTerminal:      true,
   913  						NumericExprMode: structs.NEMNumberField,
   914  						ValueIsField:    true,
   915  						Value:           "http_status",
   916  					},
   917  				},
   918  				RightValue: &structs.ValueExpr{
   919  					NumericExpr: &structs.NumericExpr{
   920  						IsTerminal:      true,
   921  						NumericExprMode: structs.NEMNumber,
   922  						ValueIsField:    false,
   923  						Value:           "400",
   924  					},
   925  				},
   926  				ValueOp: "<",
   927  			},
   928  		},
   929  	}
   930  	matchesSomeRecords[15] = true
   931  	searchResults[15] = map[string]interface{}{
   932  		"endswith": [][]*SimpleSearchExpr{
   933  			{
   934  				{
   935  					Op:           "<",
   936  					Field:        "http_status",
   937  					Values:       json.Number("400"),
   938  					ValueIsRegex: false,
   939  					ExprType:     utils.SS_DT_SIGNED_NUM,
   940  					DtypeEnclosure: &utils.DtypeEnclosure{
   941  						Dtype:       utils.SS_DT_SIGNED_NUM,
   942  						FloatVal:    float64(400),
   943  						UnsignedVal: uint64(400),
   944  						SignedVal:   int64(400),
   945  						StringVal:   "400",
   946  					},
   947  				},
   948  			},
   949  		},
   950  		"startswith": [][]*SimpleSearchExpr{
   951  			{
   952  				{
   953  					Op:           ">",
   954  					Field:        "http_status",
   955  					Values:       json.Number("300"),
   956  					ValueIsRegex: false,
   957  					ExprType:     utils.SS_DT_SIGNED_NUM,
   958  					DtypeEnclosure: &utils.DtypeEnclosure{
   959  						Dtype:       utils.SS_DT_SIGNED_NUM,
   960  						FloatVal:    float64(300),
   961  						UnsignedVal: uint64(300),
   962  						SignedVal:   int64(300),
   963  						StringVal:   "300",
   964  					},
   965  				},
   966  				{
   967  					Op:           "=",
   968  					Field:        "http_status",
   969  					Values:       json.Number("201"),
   970  					ValueIsRegex: false,
   971  					ExprType:     utils.SS_DT_SIGNED_NUM,
   972  					DtypeEnclosure: &utils.DtypeEnclosure{
   973  						Dtype:       utils.SS_DT_SIGNED_NUM,
   974  						FloatVal:    float64(201),
   975  						UnsignedVal: uint64(201),
   976  						SignedVal:   int64(201),
   977  						StringVal:   "201",
   978  					},
   979  				},
   980  			},
   981  			{
   982  				{
   983  					Op:           "=",
   984  					Field:        "http_method",
   985  					Values:       "POST",
   986  					ValueIsRegex: false,
   987  					ExprType:     utils.SS_DT_STRING,
   988  					DtypeEnclosure: &utils.DtypeEnclosure{
   989  						Dtype:     utils.SS_DT_STRING,
   990  						StringVal: "POST",
   991  					},
   992  				},
   993  			},
   994  		},
   995  	}
   996  
   997  	// CASE 16: String Search Expr: transaction city startswith="status>300 OR status=201" endswith=eval(status<400)
   998  	txnArgs16 := &structs.TransactionArguments{
   999  		Fields: []string{"city"},
  1000  		StartsWith: &structs.FilterStringExpr{
  1001  			SearchNode: &structs.ASTNode{
  1002  				OrFilterCondition: &structs.Condition{
  1003  					FilterCriteria: []*structs.FilterCriteria{
  1004  						{
  1005  							ExpressionFilter: &structs.ExpressionFilter{
  1006  								LeftInput: &structs.FilterInput{
  1007  									Expression: &structs.Expression{
  1008  										LeftInput: &structs.ExpressionInput{
  1009  											ColumnValue: nil,
  1010  											ColumnName:  "http_status",
  1011  										},
  1012  										ExpressionOp: utils.Add,
  1013  										RightInput:   nil,
  1014  									},
  1015  								},
  1016  								RightInput: &structs.FilterInput{
  1017  									Expression: &structs.Expression{
  1018  										LeftInput: &structs.ExpressionInput{
  1019  											ColumnValue: &utils.DtypeEnclosure{
  1020  												Dtype:       utils.SS_DT_UNSIGNED_NUM,
  1021  												UnsignedVal: uint64(300),
  1022  												SignedVal:   int64(300),
  1023  												FloatVal:    float64(300),
  1024  												StringVal:   "300",
  1025  											},
  1026  											ColumnName: "",
  1027  										},
  1028  										ExpressionOp: utils.Add,
  1029  										RightInput:   nil,
  1030  									},
  1031  								},
  1032  								FilterOperator: utils.GreaterThan,
  1033  							},
  1034  						},
  1035  						{
  1036  							ExpressionFilter: &structs.ExpressionFilter{
  1037  								LeftInput: &structs.FilterInput{
  1038  									Expression: &structs.Expression{
  1039  										LeftInput: &structs.ExpressionInput{
  1040  											ColumnValue: nil,
  1041  											ColumnName:  "http_status",
  1042  										},
  1043  										ExpressionOp: utils.Add,
  1044  										RightInput:   nil,
  1045  									},
  1046  								},
  1047  								RightInput: &structs.FilterInput{
  1048  									Expression: &structs.Expression{
  1049  										LeftInput: &structs.ExpressionInput{
  1050  											ColumnValue: &utils.DtypeEnclosure{
  1051  												Dtype:       utils.SS_DT_UNSIGNED_NUM,
  1052  												UnsignedVal: uint64(201),
  1053  												SignedVal:   int64(201),
  1054  												FloatVal:    float64(201),
  1055  												StringVal:   "201",
  1056  											},
  1057  											ColumnName: "",
  1058  										},
  1059  										ExpressionOp: utils.Add,
  1060  										RightInput:   nil,
  1061  									},
  1062  								},
  1063  								FilterOperator: utils.Equals,
  1064  							},
  1065  						},
  1066  					},
  1067  					NestedNodes: nil,
  1068  				},
  1069  			},
  1070  		},
  1071  		EndsWith: &structs.FilterStringExpr{
  1072  			EvalBoolExpr: &structs.BoolExpr{
  1073  				IsTerminal: true,
  1074  				LeftValue: &structs.ValueExpr{
  1075  					NumericExpr: &structs.NumericExpr{
  1076  						IsTerminal:      true,
  1077  						NumericExprMode: structs.NEMNumberField,
  1078  						ValueIsField:    true,
  1079  						Value:           "http_status",
  1080  					},
  1081  				},
  1082  				RightValue: &structs.ValueExpr{
  1083  					NumericExpr: &structs.NumericExpr{
  1084  						IsTerminal:      true,
  1085  						NumericExprMode: structs.NEMNumber,
  1086  						ValueIsField:    false,
  1087  						Value:           "400",
  1088  					},
  1089  				},
  1090  				ValueOp: "<",
  1091  			},
  1092  		},
  1093  	}
  1094  	matchesSomeRecords[16] = true
  1095  	searchResults[16] = map[string]interface{}{
  1096  		"endswith": [][]*SimpleSearchExpr{
  1097  			{
  1098  				{
  1099  					Op:           "<",
  1100  					Field:        "http_status",
  1101  					Values:       json.Number("400"),
  1102  					ValueIsRegex: false,
  1103  					ExprType:     utils.SS_DT_SIGNED_NUM,
  1104  					DtypeEnclosure: &utils.DtypeEnclosure{
  1105  						Dtype:       utils.SS_DT_SIGNED_NUM,
  1106  						FloatVal:    float64(400),
  1107  						UnsignedVal: uint64(400),
  1108  						SignedVal:   int64(400),
  1109  						StringVal:   "400",
  1110  					},
  1111  				},
  1112  			},
  1113  		},
  1114  		"startswith": [][]*SimpleSearchExpr{
  1115  			{
  1116  				{
  1117  					Op:           ">",
  1118  					Field:        "http_status",
  1119  					Values:       json.Number("300"),
  1120  					ValueIsRegex: false,
  1121  					ExprType:     utils.SS_DT_SIGNED_NUM,
  1122  					DtypeEnclosure: &utils.DtypeEnclosure{
  1123  						Dtype:       utils.SS_DT_SIGNED_NUM,
  1124  						FloatVal:    float64(300),
  1125  						UnsignedVal: uint64(300),
  1126  						SignedVal:   int64(300),
  1127  						StringVal:   "300",
  1128  					},
  1129  				},
  1130  				{
  1131  					Op:           "=",
  1132  					Field:        "http_status",
  1133  					Values:       json.Number("201"),
  1134  					ValueIsRegex: false,
  1135  					ExprType:     utils.SS_DT_SIGNED_NUM,
  1136  					DtypeEnclosure: &utils.DtypeEnclosure{
  1137  						Dtype:       utils.SS_DT_SIGNED_NUM,
  1138  						FloatVal:    float64(201),
  1139  						UnsignedVal: uint64(201),
  1140  						SignedVal:   int64(201),
  1141  						StringVal:   "201",
  1142  					},
  1143  				},
  1144  			},
  1145  		},
  1146  	}
  1147  
  1148  	return matchesSomeRecords, []*structs.TransactionArguments{txnArgs1, txnArgs2, txnArgs3, txnArgs4, txnArgs5, txnArgs6, txnArgs7, txnArgs8, txnArgs9,
  1149  		txnArgs10, txnArgs11, txnArgs12, txnArgs13, txnArgs14, txnArgs15, txnArgs16}, searchResults
  1150  }
  1151  
  1152  func Test_processTransactionsOnRecords(t *testing.T) {
  1153  
  1154  	allCols := map[string]bool{"city": true, "gender": true}
  1155  
  1156  	matchesSomeRecords, allCasesTxnArgs, searchResults := All_TestCasesForTransactionCommands()
  1157  
  1158  	for index, txnArgs := range allCasesTxnArgs {
  1159  		records := generateTestRecords(500)
  1160  		// Process Transactions
  1161  		performTransactionCommandRequest(&structs.NodeResult{}, &structs.QueryAggregators{TransactionArguments: txnArgs}, records, allCols, 1, true)
  1162  
  1163  		expectedCols := map[string]bool{"duration": true, "event": true, "eventcount": true, "timestamp": true}
  1164  
  1165  		for _, field := range txnArgs.Fields {
  1166  			expectedCols[field] = true
  1167  		}
  1168  
  1169  		assert.Equal(t, expectedCols, allCols)
  1170  
  1171  		// Check if the number of records is positive or negative
  1172  		assert.Equal(t, matchesSomeRecords[index+1], len(records) > 0)
  1173  
  1174  		for _, record := range records {
  1175  			assert.Equal(t, record["timestamp"], uint64(1659874108987))
  1176  			assert.Equal(t, record["duration"], uint64(0))
  1177  
  1178  			events := record["event"].([]map[string]interface{})
  1179  
  1180  			initFields := []string{}
  1181  
  1182  			for ind, eventMap := range events {
  1183  				fields := []string{}
  1184  
  1185  				for _, field := range txnArgs.Fields {
  1186  					fields = append(fields, eventMap[field].(string))
  1187  				}
  1188  
  1189  				// Check if the fields are same for all events by assigning the first event's fields to initFields
  1190  				if ind == 0 {
  1191  					initFields = fields
  1192  				}
  1193  
  1194  				assert.Equal(t, fields, initFields)
  1195  
  1196  				if txnArgs.StartsWith != nil {
  1197  					if ind == 0 {
  1198  						resultData, exists := getResultData(index+1, "startswith", searchResults)
  1199  						if txnArgs.StartsWith.StringValue != "" {
  1200  							assert.Equal(t, eventMap["http_method"], txnArgs.StartsWith.StringValue)
  1201  						} else if txnArgs.StartsWith.EvalBoolExpr != nil {
  1202  							if exists {
  1203  								valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr))
  1204  								assert.True(t, valid)
  1205  							}
  1206  						} else if txnArgs.StartsWith.SearchNode != nil {
  1207  							if exists {
  1208  								valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr))
  1209  								assert.True(t, valid)
  1210  							}
  1211  						}
  1212  					}
  1213  				}
  1214  
  1215  				if txnArgs.EndsWith != nil {
  1216  					if ind == len(events)-1 {
  1217  						resultData, exists := getResultData(index+1, "endswith", searchResults)
  1218  						if txnArgs.EndsWith.StringValue != "" {
  1219  							assert.Equal(t, eventMap["http_method"], txnArgs.EndsWith.StringValue)
  1220  						} else if txnArgs.EndsWith.EvalBoolExpr != nil {
  1221  							if exists {
  1222  								valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr))
  1223  								assert.True(t, valid)
  1224  							}
  1225  						} else if txnArgs.EndsWith.SearchNode != nil {
  1226  							if exists {
  1227  								valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr))
  1228  								assert.True(t, valid)
  1229  							}
  1230  						}
  1231  					}
  1232  				}
  1233  
  1234  			}
  1235  		}
  1236  	}
  1237  
  1238  }
  1239  
  1240  func getResultData(resultIndex int, resultType string, resultData map[int]map[string]interface{}) (interface{}, bool) {
  1241  	resultDataMap, exists := resultData[resultIndex]
  1242  	if exists {
  1243  		data, exists := resultDataMap[resultType]
  1244  		return data, exists
  1245  	} else {
  1246  		return nil, false
  1247  	}
  1248  }
  1249  
  1250  func validateSearchString(searchTerm *SimpleSearchExpr, eventMap map[string]interface{}) bool {
  1251  	fieldValue, exists := eventMap[searchTerm.Field]
  1252  	if !exists {
  1253  		return false
  1254  	}
  1255  
  1256  	return conditionMatch(fieldValue, searchTerm.Op, searchTerm.Values)
  1257  }
  1258  
  1259  func validateSearchExpr(eventMap map[string]interface{}, resultData [][]*SimpleSearchExpr) bool {
  1260  	for _, resultAnd := range resultData {
  1261  		valid := false
  1262  		for _, resultOr := range resultAnd {
  1263  			if validateSearchString(resultOr, eventMap) {
  1264  				valid = true
  1265  				break
  1266  			}
  1267  		}
  1268  		if !valid {
  1269  			return false
  1270  		}
  1271  	}
  1272  	return true
  1273  }
  1274  
  1275  func Test_performValueColRequestWithoutGroupBy_VEMNumericExpr(t *testing.T) {
  1276  
  1277  	// Query 1: * | eval rlatitude=round(latitude, 2)
  1278  	letColReq := &structs.LetColumnsRequest{
  1279  		ValueColRequest: &structs.ValueExpr{
  1280  			ValueExprMode: structs.VEMNumericExpr,
  1281  			NumericExpr: &structs.NumericExpr{
  1282  				NumericExprMode: structs.NEMNumericExpr,
  1283  				Op:              "round",
  1284  				Left: &structs.NumericExpr{
  1285  					NumericExprMode: structs.NEMNumberField,
  1286  					IsTerminal:      true,
  1287  					ValueIsField:    true,
  1288  					Value:           "latitude",
  1289  				},
  1290  				Right: &structs.NumericExpr{
  1291  					NumericExprMode: structs.NEMNumber,
  1292  					IsTerminal:      true,
  1293  					ValueIsField:    false,
  1294  					Value:           "2",
  1295  				},
  1296  			},
  1297  		},
  1298  		NewColName: "rlatitude",
  1299  	}
  1300  
  1301  	records := generateTestRecords(500)
  1302  	finalCols := getFinalColsForGeneratedTestRecords()
  1303  
  1304  	// Perform the value column request
  1305  	err := performValueColRequest(&structs.NodeResult{}, &structs.QueryAggregators{}, letColReq, records, finalCols)
  1306  	assert.Nil(t, err)
  1307  
  1308  	// Check if the new column is added to the records
  1309  	assert.True(t, finalCols["rlatitude"])
  1310  
  1311  	for _, record := range records {
  1312  		assert.True(t, record["rlatitude"] != nil)
  1313  		valueStr := fmt.Sprintf("%.2f", record["latitude"].(float64))
  1314  		splitValue := strings.Split(valueStr, ".")
  1315  		if len(splitValue) > 1 {
  1316  			assert.Equal(t, len(splitValue[1]), 2)
  1317  		}
  1318  	}
  1319  
  1320  }
  1321  
  1322  func Test_performValueColRequestWithoutGroupBy_VEMConditionExpr(t *testing.T) {
  1323  	// Query: * |  eval http_status_mod=if(in(http_status, 400,  500), "Failure", http_status)
  1324  	letColReq := &structs.LetColumnsRequest{
  1325  		ValueColRequest: &structs.ValueExpr{
  1326  			ValueExprMode: structs.VEMConditionExpr,
  1327  			ConditionExpr: &structs.ConditionExpr{
  1328  				Op: "if",
  1329  				BoolExpr: &structs.BoolExpr{
  1330  					IsTerminal: true,
  1331  					LeftValue: &structs.ValueExpr{
  1332  						NumericExpr: &structs.NumericExpr{
  1333  							NumericExprMode: structs.NEMNumberField,
  1334  							IsTerminal:      true,
  1335  							ValueIsField:    true,
  1336  							Value:           "http_status",
  1337  						},
  1338  					},
  1339  					ValueOp: "in",
  1340  					ValueList: []*structs.ValueExpr{
  1341  						{
  1342  							NumericExpr: &structs.NumericExpr{
  1343  								NumericExprMode: structs.NEMNumber,
  1344  								IsTerminal:      true,
  1345  								ValueIsField:    false,
  1346  								Value:           "400",
  1347  							},
  1348  						},
  1349  						{
  1350  							NumericExpr: &structs.NumericExpr{
  1351  								NumericExprMode: structs.NEMNumber,
  1352  								IsTerminal:      true,
  1353  								ValueIsField:    false,
  1354  								Value:           "500",
  1355  							},
  1356  						},
  1357  					},
  1358  				},
  1359  				TrueValue: &structs.ValueExpr{
  1360  					ValueExprMode: structs.VEMStringExpr,
  1361  					StringExpr: &structs.StringExpr{
  1362  						RawString: "Failure",
  1363  					},
  1364  				},
  1365  				FalseValue: &structs.ValueExpr{
  1366  					NumericExpr: &structs.NumericExpr{
  1367  						NumericExprMode: structs.NEMNumberField,
  1368  						IsTerminal:      true,
  1369  						ValueIsField:    true,
  1370  						Value:           "http_status",
  1371  					},
  1372  				},
  1373  			},
  1374  		},
  1375  		NewColName: "http_status_mod",
  1376  	}
  1377  
  1378  	records := generateTestRecords(500)
  1379  	finalCols := getFinalColsForGeneratedTestRecords()
  1380  
  1381  	// Perform the value column request
  1382  	err := performValueColRequest(&structs.NodeResult{}, &structs.QueryAggregators{}, letColReq, records, finalCols)
  1383  	assert.Nil(t, err)
  1384  
  1385  	// Check if the new column is added to the records
  1386  	assert.True(t, finalCols["http_status_mod"])
  1387  
  1388  	for _, record := range records {
  1389  		assert.True(t, record["http_status_mod"] != nil)
  1390  		httpStatus := record["http_status"].(int64)
  1391  		if httpStatus == 400 || httpStatus == 500 {
  1392  			assert.Equal(t, "Failure", record["http_status_mod"])
  1393  		} else {
  1394  			assert.Equal(t, fmt.Sprint(httpStatus), record["http_status_mod"])
  1395  		}
  1396  	}
  1397  }
  1398  
  1399  func Test_performValueColRequestWithoutGroupBy_VEMStringExpr(t *testing.T) {
  1400  	// Query: * | eval country_city=country.":-".city
  1401  	letColReq := &structs.LetColumnsRequest{
  1402  		ValueColRequest: &structs.ValueExpr{
  1403  			ValueExprMode: structs.VEMStringExpr,
  1404  			StringExpr: &structs.StringExpr{
  1405  				StringExprMode: structs.SEMConcatExpr,
  1406  				ConcatExpr: &structs.ConcatExpr{
  1407  					Atoms: []*structs.ConcatAtom{
  1408  						{
  1409  							IsField: true,
  1410  							Value:   "country",
  1411  						},
  1412  						{
  1413  							IsField: false,
  1414  							Value:   ":-",
  1415  						},
  1416  						{
  1417  							IsField: true,
  1418  							Value:   "city",
  1419  						},
  1420  					},
  1421  				},
  1422  			},
  1423  		},
  1424  		NewColName: "country_city",
  1425  	}
  1426  
  1427  	records := generateTestRecords(500)
  1428  	finalCols := getFinalColsForGeneratedTestRecords()
  1429  
  1430  	// Perform the value column request
  1431  	err := performValueColRequest(&structs.NodeResult{}, &structs.QueryAggregators{}, letColReq, records, finalCols)
  1432  
  1433  	assert.Nil(t, err)
  1434  
  1435  	// Check if the new column is added to the records
  1436  	assert.True(t, finalCols["country_city"])
  1437  
  1438  	for _, record := range records {
  1439  		assert.True(t, record["country_city"] != nil)
  1440  		country := record["country"].(string)
  1441  		city := record["city"].(string)
  1442  		assert.Equal(t, country+":-"+city, record["country_city"])
  1443  	}
  1444  }
  1445  
  1446  func Test_getColumnsToKeepAndRemove(t *testing.T) {
  1447  	tests := []struct {
  1448  		name             string
  1449  		cols             []string
  1450  		wildcardCols     []string
  1451  		keepMatches      bool
  1452  		wantIndices      []int
  1453  		wantColsToKeep   []string
  1454  		wantColsToRemove []string
  1455  	}{
  1456  		{
  1457  			name:             "No wildcards, keepMatches true",
  1458  			cols:             []string{"id", "name", "email"},
  1459  			wildcardCols:     []string{},
  1460  			keepMatches:      true,
  1461  			wantIndices:      []int{},
  1462  			wantColsToKeep:   []string{},
  1463  			wantColsToRemove: []string{"id", "name", "email"},
  1464  		},
  1465  		{
  1466  			name:             "No wildcards, keepMatches false",
  1467  			cols:             []string{"id", "name", "email"},
  1468  			wildcardCols:     []string{},
  1469  			keepMatches:      false,
  1470  			wantIndices:      []int{0, 1, 2},
  1471  			wantColsToKeep:   []string{"id", "name", "email"},
  1472  			wantColsToRemove: []string{},
  1473  		},
  1474  		{
  1475  			name:             "Exact match one wildcard, keepMatches true",
  1476  			cols:             []string{"id", "name", "email"},
  1477  			wildcardCols:     []string{"name"},
  1478  			keepMatches:      true,
  1479  			wantIndices:      []int{1},
  1480  			wantColsToKeep:   []string{"name"},
  1481  			wantColsToRemove: []string{"id", "email"},
  1482  		},
  1483  		{
  1484  			name:             "Wildcard matches multiple columns, keepMatches true",
  1485  			cols:             []string{"user_id", "username", "user_email", "age"},
  1486  			wildcardCols:     []string{"user_*"},
  1487  			keepMatches:      true,
  1488  			wantIndices:      []int{0, 2},
  1489  			wantColsToKeep:   []string{"user_id", "user_email"},
  1490  			wantColsToRemove: []string{"username", "age"},
  1491  		},
  1492  		{
  1493  			name:             "Wildcard matches none, keepMatches false",
  1494  			cols:             []string{"id", "name", "email"},
  1495  			wildcardCols:     []string{"user_*"},
  1496  			keepMatches:      false,
  1497  			wantIndices:      []int{0, 1, 2},
  1498  			wantColsToKeep:   []string{"id", "name", "email"},
  1499  			wantColsToRemove: []string{},
  1500  		},
  1501  		{
  1502  			name:             "Multiple wildcards with overlaps, keepMatches true",
  1503  			cols:             []string{"user_id", "admin_id", "username", "email"},
  1504  			wildcardCols:     []string{"user_*", "*_id"},
  1505  			keepMatches:      true,
  1506  			wantIndices:      []int{0, 1},
  1507  			wantColsToKeep:   []string{"user_id", "admin_id"},
  1508  			wantColsToRemove: []string{"username", "email"},
  1509  		},
  1510  		{
  1511  			name:             "Empty cols, keepMatches true",
  1512  			cols:             []string{},
  1513  			wildcardCols:     []string{"user_*"},
  1514  			keepMatches:      true,
  1515  			wantIndices:      []int{},
  1516  			wantColsToKeep:   []string{},
  1517  			wantColsToRemove: []string{},
  1518  		},
  1519  		{
  1520  			name:             "Wildcard matches all, keepMatches false",
  1521  			cols:             []string{"user_id", "user_name", "user_email"},
  1522  			wildcardCols:     []string{"user_*"},
  1523  			keepMatches:      false,
  1524  			wantIndices:      []int{},
  1525  			wantColsToKeep:   []string{},
  1526  			wantColsToRemove: []string{"user_id", "user_name", "user_email"},
  1527  		},
  1528  		{
  1529  			name:             "Complex wildcards, partial matches",
  1530  			cols:             []string{"user_id", "admin_id", "username", "user_profile", "user_email", "age"},
  1531  			wildcardCols:     []string{"user_*", "*name"},
  1532  			keepMatches:      true,
  1533  			wantIndices:      []int{0, 2, 3, 4},
  1534  			wantColsToKeep:   []string{"user_id", "username", "user_profile", "user_email"},
  1535  			wantColsToRemove: []string{"admin_id", "age"},
  1536  		},
  1537  	}
  1538  
  1539  	for _, tt := range tests {
  1540  		t.Run(tt.name, func(t *testing.T) {
  1541  			gotIndices, gotColsToKeep, gotColsToRemove := getColumnsToKeepAndRemove(tt.cols, tt.wildcardCols, tt.keepMatches)
  1542  
  1543  			assert.Equal(t, tt.wantIndices, gotIndices)
  1544  			assert.Equal(t, tt.wantColsToKeep, gotColsToKeep)
  1545  			assert.Equal(t, tt.wantColsToRemove, gotColsToRemove)
  1546  		})
  1547  	}
  1548  }