github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/structs/evaluationstructs_test.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package structs
    18  
    19  import (
    20  	"testing"
    21  	"time"
    22  
    23  	segutils "github.com/siglens/siglens/pkg/segment/utils"
    24  	"github.com/siglens/siglens/pkg/utils"
    25  	"github.com/stretchr/testify/assert"
    26  )
    27  
    28  func Test_ConcatExpr(t *testing.T) {
    29  	expr := &ConcatExpr{
    30  		Atoms: []*ConcatAtom{
    31  			{IsField: true, Value: "FieldWithStrings"},
    32  			{IsField: false, Value: " and "},
    33  			{IsField: true, Value: "FieldWithNumbers"},
    34  		},
    35  	}
    36  
    37  	// Test GetFields()
    38  	assert.Equal(t, expr.GetFields(), []string{"FieldWithStrings", "FieldWithNumbers"})
    39  
    40  	// Test Evaluate()
    41  	fieldToValue := make(map[string]segutils.CValueEnclosure)
    42  	fieldToValue["FieldWithStrings"] = segutils.CValueEnclosure{
    43  		Dtype: segutils.SS_DT_STRING,
    44  		CVal:  "testing",
    45  	}
    46  	fieldToValue["FieldWithNumbers"] = segutils.CValueEnclosure{
    47  		Dtype: segutils.SS_DT_SIGNED_NUM,
    48  		CVal:  int64(123),
    49  	}
    50  
    51  	value, err := expr.Evaluate(fieldToValue)
    52  	assert.Nil(t, err)
    53  	assert.Equal(t, value, "testing and 123")
    54  
    55  	// When fieldToValue is missing fields, Evaluate() should error.
    56  	delete(fieldToValue, "FieldWithNumbers")
    57  	_, err = expr.Evaluate(fieldToValue)
    58  	assert.NotNil(t, err)
    59  }
    60  
    61  func Test_NumericExpr(t *testing.T) {
    62  	exprA := &NumericExpr{
    63  		IsTerminal:   true,
    64  		ValueIsField: false,
    65  		Value:        "5",
    66  	}
    67  	exprB := &NumericExpr{
    68  		IsTerminal:      true,
    69  		ValueIsField:    true,
    70  		Value:           "Max",
    71  		NumericExprMode: NEMNumberField,
    72  	}
    73  	exprC := &NumericExpr{
    74  		IsTerminal: false,
    75  		Op:         "-",
    76  		Left:       exprB,
    77  		Right:      exprA,
    78  	}
    79  	exprD := &NumericExpr{
    80  		IsTerminal:      true,
    81  		ValueIsField:    true,
    82  		Value:           "Min",
    83  		NumericExprMode: NEMNumberField,
    84  	}
    85  	exprE := &NumericExpr{
    86  		IsTerminal: false,
    87  		Op:         "/",
    88  		Left:       exprC,
    89  		Right:      exprD,
    90  	}
    91  
    92  	// Test GetFields()
    93  	assert.Equal(t, exprA.GetFields(), []string{})
    94  
    95  	assert.Equal(t, exprB.GetFields(), []string{"Max"})
    96  	assert.Equal(t, exprC.GetFields(), []string{"Max"})
    97  	assert.Equal(t, exprD.GetFields(), []string{"Min"})
    98  
    99  	eFields := exprE.GetFields()
   100  	assert.True(t, utils.SliceContainsString(eFields, "Min"))
   101  
   102  	assert.True(t, utils.SliceContainsString(eFields, "Max"))
   103  
   104  	// Test Evaluate()
   105  	fieldToValue := make(map[string]segutils.CValueEnclosure)
   106  	fieldToValue["Min"] = segutils.CValueEnclosure{
   107  		Dtype: segutils.SS_DT_FLOAT,
   108  		CVal:  float64(12),
   109  	}
   110  
   111  	fieldToValue["Max"] = segutils.CValueEnclosure{
   112  		Dtype: segutils.SS_DT_SIGNED_NUM,
   113  		CVal:  int64(62),
   114  	}
   115  
   116  	value, err := exprA.Evaluate(fieldToValue)
   117  	assert.Nil(t, err)
   118  	assert.Equal(t, value, float64(5))
   119  
   120  	value, err = exprB.Evaluate(fieldToValue)
   121  
   122  	assert.Nil(t, err)
   123  	assert.Equal(t, value, float64(62))
   124  
   125  	value, err = exprC.Evaluate(fieldToValue)
   126  	assert.Nil(t, err)
   127  	assert.Equal(t, value, float64(62-5))
   128  
   129  	value, err = exprD.Evaluate(fieldToValue)
   130  	assert.Nil(t, err)
   131  	assert.Equal(t, value, float64(12))
   132  
   133  	value, err = exprE.Evaluate(fieldToValue)
   134  	assert.Nil(t, err)
   135  	assert.Equal(t, value, float64(62-5)/12)
   136  
   137  	// When fieldToValue is missing fields, Evaluate() should error.
   138  	delete(fieldToValue, "Max")
   139  	_, err = exprE.Evaluate(fieldToValue)
   140  	assert.NotNil(t, err)
   141  
   142  	multiplierExpr := &NumericExpr{
   143  		IsTerminal:   true,
   144  		ValueIsField: false,
   145  		Value:        "3.14",
   146  	}
   147  	httpStatusExpr := &NumericExpr{
   148  		IsTerminal:      true,
   149  		ValueIsField:    true,
   150  		Value:           "http_status",
   151  		NumericExprMode: NEMNumberField,
   152  	}
   153  	productExpr := &NumericExpr{
   154  		IsTerminal: false,
   155  		Op:         "*",
   156  		Left:       multiplierExpr,
   157  		Right:      httpStatusExpr,
   158  	}
   159  	exactExpr := &NumericExpr{
   160  		IsTerminal: false,
   161  		Op:         "exact",
   162  		Left:       productExpr,
   163  	}
   164  	assert.Equal(t, exactExpr.GetFields(), []string{"http_status"})
   165  
   166  	fieldToValue["http_status"] = segutils.CValueEnclosure{
   167  		Dtype: segutils.SS_DT_SIGNED_NUM,
   168  		CVal:  int64(200),
   169  	}
   170  
   171  	value, err = exactExpr.Evaluate(fieldToValue)
   172  	assert.Nil(t, err)
   173  	assert.Equal(t, value, float64(628))
   174  
   175  	expExpr := &NumericExpr{
   176  		IsTerminal: false,
   177  		Op:         "exp",
   178  		Left: &NumericExpr{
   179  			IsTerminal:   true,
   180  			ValueIsField: false,
   181  			Value:        "3",
   182  		},
   183  	}
   184  
   185  	assert.Equal(t, expExpr.GetFields(), []string{})
   186  
   187  	value, err = expExpr.Evaluate(fieldToValue)
   188  	assert.Nil(t, err)
   189  	assert.Equal(t, value, 20.085536923187668)
   190  
   191  	nowExpr := &NumericExpr{
   192  		NumericExprMode: NEMNumber,
   193  		IsTerminal:      true,
   194  		Op:              "now",
   195  	}
   196  	assert.Equal(t, nowExpr.GetFields(), []string{})
   197  
   198  	// Test Evaluate()
   199  	fieldToValue = make(map[string]segutils.CValueEnclosure)
   200  
   201  	value, err = nowExpr.Evaluate(fieldToValue)
   202  	assert.Nil(t, err)
   203  	currentTimestamp := time.Now().Unix()
   204  
   205  	assert.InDelta(t, currentTimestamp, value, 1, "The evaluated timestamp is not within the expected range")
   206  
   207  	strToNumber :=
   208  		&NumericExpr{
   209  			NumericExprMode: NEMNumericExpr,
   210  			IsTerminal:      false,
   211  			Op:              "tonumber",
   212  			Right: &NumericExpr{
   213  				NumericExprMode: NEMNumber,
   214  				IsTerminal:      true,
   215  				ValueIsField:    false,
   216  				Value:           "16",
   217  			},
   218  			Val: &StringExpr{
   219  				StringExprMode: SEMRawString,
   220  				RawString:      "0A4",
   221  			},
   222  		}
   223  	assert.Equal(t, strToNumber.GetFields(), []string{})
   224  
   225  	value, err = strToNumber.Evaluate(fieldToValue)
   226  	assert.Nil(t, err)
   227  	assert.Equal(t, value, float64(164))
   228  
   229  }
   230  
   231  func Test_ValueExpr(t *testing.T) {
   232  	numericExpr := &NumericExpr{
   233  		IsTerminal: false,
   234  		Op:         "-",
   235  		Left: &NumericExpr{
   236  			IsTerminal:      true,
   237  			ValueIsField:    true,
   238  			Value:           "Max",
   239  			NumericExprMode: NEMNumberField,
   240  		},
   241  		Right: &NumericExpr{
   242  			IsTerminal:   true,
   243  			ValueIsField: false,
   244  			Value:        "5",
   245  		},
   246  	}
   247  
   248  	concatExpr := &ConcatExpr{
   249  		Atoms: []*ConcatAtom{
   250  			{IsField: true, Value: "FieldWithStrings"},
   251  			{IsField: false, Value: " and "},
   252  			{IsField: true, Value: "FieldWithNumbers"},
   253  		},
   254  	}
   255  
   256  	valueExprA := &ValueExpr{
   257  		ValueExprMode: VEMNumericExpr,
   258  		NumericExpr:   numericExpr,
   259  	}
   260  
   261  	valueExprB := &ValueExpr{
   262  		ValueExprMode: VEMStringExpr,
   263  		StringExpr: &StringExpr{
   264  			StringExprMode: SEMConcatExpr,
   265  			ConcatExpr:     concatExpr,
   266  		},
   267  	}
   268  
   269  	valueExprC := &ValueExpr{
   270  		ValueExprMode: VEMStringExpr,
   271  		StringExpr: &StringExpr{
   272  			StringExprMode: SEMConcatExpr,
   273  			ConcatExpr: &ConcatExpr{
   274  				Atoms: []*ConcatAtom{
   275  					{Value: "hello"},
   276  				},
   277  			},
   278  		},
   279  	}
   280  	valueExprD := &ValueExpr{
   281  		ValueExprMode: VEMNumericExpr,
   282  		NumericExpr: &NumericExpr{
   283  
   284  			IsTerminal:   true,
   285  			ValueIsField: false,
   286  			Value:        "99",
   287  		},
   288  	}
   289  
   290  	valueExprE := &ValueExpr{
   291  		ValueExprMode: VEMNumericExpr,
   292  		NumericExpr: &NumericExpr{
   293  			IsTerminal:   true,
   294  			ValueIsField: true,
   295  
   296  			Value:           "Seconds",
   297  			NumericExprMode: NEMNumberField,
   298  		},
   299  	}
   300  
   301  	// Test GetFields()
   302  	assert.Equal(t, valueExprA.GetFields(), []string{"Max"})
   303  	assert.Equal(t, valueExprB.GetFields(), []string{"FieldWithStrings", "FieldWithNumbers"})
   304  	assert.Equal(t, valueExprC.GetFields(), []string{})
   305  	assert.Equal(t, valueExprD.GetFields(), []string{})
   306  	assert.Equal(t, valueExprE.GetFields(), []string{"Seconds"})
   307  
   308  	// Test Evaluate()
   309  	fieldToValue := make(map[string]segutils.CValueEnclosure)
   310  	fieldToValue["Max"] = segutils.CValueEnclosure{
   311  		Dtype: segutils.SS_DT_FLOAT,
   312  		CVal:  float64(62),
   313  	}
   314  	fieldToValue["Seconds"] = segutils.CValueEnclosure{
   315  		Dtype: segutils.SS_DT_SIGNED_NUM,
   316  		CVal:  int64(42),
   317  	}
   318  	fieldToValue["FieldWithStrings"] = segutils.CValueEnclosure{
   319  		Dtype: segutils.SS_DT_STRING,
   320  		CVal:  "testing",
   321  	}
   322  	fieldToValue["FieldWithNumbers"] = segutils.CValueEnclosure{
   323  		Dtype: segutils.SS_DT_SIGNED_NUM,
   324  		CVal:  int64(123),
   325  	}
   326  
   327  	var valueFloat float64
   328  	var valueStr string
   329  	var err error
   330  	valueFloat, err = valueExprA.EvaluateToFloat(fieldToValue)
   331  	assert.Nil(t, err)
   332  	assert.Equal(t, valueFloat, float64(62-5))
   333  	valueStr, err = valueExprA.EvaluateToString(fieldToValue)
   334  	assert.Nil(t, err)
   335  	assert.Equal(t, valueStr, "57")
   336  
   337  	_, err = valueExprB.EvaluateToFloat(fieldToValue)
   338  	assert.NotNil(t, err)
   339  	valueStr, err = valueExprB.EvaluateToString(fieldToValue)
   340  	assert.Nil(t, err)
   341  	assert.Equal(t, valueStr, "testing and 123")
   342  
   343  	_, err = valueExprC.EvaluateToFloat(fieldToValue)
   344  	assert.NotNil(t, err)
   345  	valueStr, err = valueExprC.EvaluateToString(fieldToValue)
   346  	assert.Nil(t, err)
   347  	assert.Equal(t, valueStr, "hello")
   348  
   349  	valueFloat, err = valueExprD.EvaluateToFloat(fieldToValue)
   350  	assert.Nil(t, err)
   351  	assert.Equal(t, valueFloat, float64(99))
   352  	valueStr, err = valueExprD.EvaluateToString(fieldToValue)
   353  	assert.Nil(t, err)
   354  	assert.Equal(t, valueStr, "99")
   355  
   356  	valueFloat, err = valueExprE.EvaluateToFloat(fieldToValue)
   357  	assert.Nil(t, err)
   358  	assert.Equal(t, valueFloat, float64(42))
   359  	valueStr, err = valueExprE.EvaluateToString(fieldToValue)
   360  	assert.Nil(t, err)
   361  	assert.Equal(t, valueStr, "42")
   362  
   363  	// When fieldToValue is missing fields, Evaluate() should error.
   364  	fieldToValue = make(map[string]segutils.CValueEnclosure)
   365  	_, err = valueExprA.EvaluateToFloat(fieldToValue)
   366  	assert.NotNil(t, err)
   367  	_, err = valueExprA.EvaluateToString(fieldToValue)
   368  	assert.NotNil(t, err)
   369  
   370  	_, err = valueExprB.EvaluateToFloat(fieldToValue)
   371  	assert.NotNil(t, err)
   372  	_, err = valueExprA.EvaluateToString(fieldToValue)
   373  	assert.NotNil(t, err)
   374  
   375  	_, err = valueExprE.EvaluateToFloat(fieldToValue)
   376  	assert.NotNil(t, err)
   377  	_, err = valueExprA.EvaluateToString(fieldToValue)
   378  	assert.NotNil(t, err)
   379  }
   380  
   381  func Test_BoolExpr(t *testing.T) {
   382  	valueExprA := &ValueExpr{
   383  		ValueExprMode: VEMNumericExpr,
   384  		NumericExpr: &NumericExpr{
   385  			IsTerminal: false,
   386  			Op:         "-",
   387  			Left: &NumericExpr{
   388  				IsTerminal:      true,
   389  				ValueIsField:    true,
   390  				Value:           "Max",
   391  				NumericExprMode: NEMNumberField,
   392  			},
   393  			Right: &NumericExpr{
   394  				IsTerminal:   true,
   395  				ValueIsField: false,
   396  				Value:        "5",
   397  			},
   398  		},
   399  	}
   400  
   401  	valueExprB := &ValueExpr{
   402  		ValueExprMode: VEMStringExpr,
   403  		StringExpr: &StringExpr{
   404  			StringExprMode: SEMConcatExpr,
   405  			ConcatExpr: &ConcatExpr{
   406  				Atoms: []*ConcatAtom{
   407  					{IsField: true, Value: "FieldWithStrings"},
   408  					{IsField: false, Value: " and "},
   409  					{IsField: true, Value: "FieldWithNumbers"},
   410  				},
   411  			},
   412  		},
   413  	}
   414  
   415  	valueExprC := &ValueExpr{
   416  		ValueExprMode: VEMStringExpr,
   417  		StringExpr: &StringExpr{
   418  			StringExprMode: SEMConcatExpr,
   419  			ConcatExpr: &ConcatExpr{
   420  
   421  				Atoms: []*ConcatAtom{
   422  					{Value: "hello"},
   423  				},
   424  			},
   425  		},
   426  	}
   427  
   428  	valueExprD := &ValueExpr{
   429  		ValueExprMode: VEMNumericExpr,
   430  		NumericExpr: &NumericExpr{
   431  
   432  			IsTerminal:   true,
   433  			ValueIsField: false,
   434  			Value:        "99",
   435  		},
   436  	}
   437  
   438  	boolExprA := &BoolExpr{
   439  		IsTerminal: true,
   440  		ValueOp:    "<",
   441  		LeftValue:  valueExprA,
   442  		RightValue: valueExprD,
   443  	}
   444  
   445  	boolExprB := &BoolExpr{
   446  		IsTerminal: true,
   447  		ValueOp:    "=",
   448  		LeftValue:  valueExprB,
   449  		RightValue: valueExprC,
   450  	}
   451  
   452  	boolExprC := &BoolExpr{
   453  		IsTerminal: false,
   454  		BoolOp:     BoolOpOr,
   455  		LeftBool:   boolExprA,
   456  		RightBool:  boolExprB,
   457  	}
   458  
   459  	boolExprD := &BoolExpr{
   460  		IsTerminal: false,
   461  		BoolOp:     BoolOpAnd,
   462  		LeftBool:   boolExprA,
   463  		RightBool:  boolExprB,
   464  	}
   465  
   466  	boolExprE := &BoolExpr{
   467  		IsTerminal: false,
   468  		BoolOp:     BoolOpNot,
   469  		LeftBool:   boolExprA,
   470  	}
   471  
   472  	// Test GetFields()
   473  	assert.Equal(t, boolExprA.GetFields(), []string{"Max"})
   474  	assert.Equal(t, boolExprB.GetFields(), []string{"FieldWithStrings", "FieldWithNumbers"})
   475  
   476  	cFields := boolExprC.GetFields()
   477  	assert.True(t, utils.SliceContainsString(cFields, "Max"))
   478  	assert.True(t, utils.SliceContainsString(cFields, "FieldWithStrings"))
   479  	assert.True(t, utils.SliceContainsString(cFields, "FieldWithNumbers"))
   480  
   481  	dFields := boolExprD.GetFields()
   482  	assert.True(t, utils.SliceContainsString(dFields, "Max"))
   483  	assert.True(t, utils.SliceContainsString(dFields, "FieldWithStrings"))
   484  	assert.True(t, utils.SliceContainsString(dFields, "FieldWithNumbers"))
   485  
   486  	assert.Equal(t, boolExprE.GetFields(), []string{"Max"})
   487  
   488  	// Test Evaluate()
   489  	fieldToValue := make(map[string]segutils.CValueEnclosure)
   490  	fieldToValue["Max"] = segutils.CValueEnclosure{
   491  		Dtype: segutils.SS_DT_FLOAT,
   492  		CVal:  float64(62),
   493  	}
   494  	fieldToValue["FieldWithStrings"] = segutils.CValueEnclosure{
   495  		Dtype: segutils.SS_DT_STRING,
   496  		CVal:  "testing",
   497  	}
   498  	fieldToValue["FieldWithNumbers"] = segutils.CValueEnclosure{
   499  		Dtype: segutils.SS_DT_SIGNED_NUM,
   500  		CVal:  int64(123),
   501  	}
   502  
   503  	value, err := boolExprA.Evaluate(fieldToValue)
   504  
   505  	assert.Nil(t, err)
   506  	assert.Equal(t, value, true)
   507  
   508  	value, err = boolExprB.Evaluate(fieldToValue)
   509  	assert.Nil(t, err)
   510  	assert.Equal(t, value, false)
   511  
   512  	value, err = boolExprC.Evaluate(fieldToValue)
   513  	assert.Nil(t, err)
   514  	assert.Equal(t, value, true)
   515  
   516  	value, err = boolExprD.Evaluate(fieldToValue)
   517  	assert.Nil(t, err)
   518  	assert.Equal(t, value, false)
   519  
   520  	value, err = boolExprE.Evaluate(fieldToValue)
   521  	assert.Nil(t, err)
   522  	assert.Equal(t, value, false)
   523  
   524  	// In a terminal node, the left and right ValueExpr must both be strings or both be floats.
   525  	boolExprBadValueExprTypes := &BoolExpr{
   526  		IsTerminal: true,
   527  		ValueOp:    "!=",
   528  		LeftValue:  valueExprA, // Evaluates to float
   529  		RightValue: valueExprB, // Evaluates to string
   530  	}
   531  
   532  	value, err = boolExprBadValueExprTypes.Evaluate(fieldToValue)
   533  	assert.Nil(t, err)
   534  	assert.Equal(t, value, true)
   535  
   536  	// In a terminal node with string ValueExprs, the ValueOp cannot be an inequality.
   537  	boolExprBadOpForStringValues := &BoolExpr{
   538  		IsTerminal: true,
   539  		ValueOp:    "<",
   540  		LeftValue:  valueExprB, // Evaluates to string
   541  		RightValue: valueExprC, // Evaluates to string
   542  	}
   543  
   544  	_, err = boolExprBadOpForStringValues.Evaluate(fieldToValue)
   545  	assert.NotNil(t, err)
   546  
   547  	// When fieldToValue is missing fields, Evaluate() should error.
   548  	delete(fieldToValue, "Max")
   549  	delete(fieldToValue, "FieldWithNumbers")
   550  	_, err = boolExprA.Evaluate(fieldToValue)
   551  	assert.NotNil(t, err)
   552  
   553  	_, err = boolExprB.Evaluate(fieldToValue)
   554  	assert.NotNil(t, err)
   555  
   556  	_, err = boolExprC.Evaluate(fieldToValue)
   557  	assert.NotNil(t, err)
   558  
   559  	_, err = boolExprD.Evaluate(fieldToValue)
   560  	assert.NotNil(t, err)
   561  
   562  	_, err = boolExprE.Evaluate(fieldToValue)
   563  	assert.NotNil(t, err)
   564  }
   565  
   566  func Test_ConditionExpr(t *testing.T) {
   567  
   568  	boolExpr :=
   569  		&BoolExpr{
   570  			IsTerminal: true,
   571  			LeftValue: &ValueExpr{
   572  				ValueExprMode: VEMNumericExpr,
   573  				NumericExpr: &NumericExpr{
   574  					NumericExprMode: NEMNumberField,
   575  					IsTerminal:      true,
   576  					ValueIsField:    true,
   577  					Value:           "state",
   578  				},
   579  			},
   580  			ValueOp: "in",
   581  			ValueList: []*ValueExpr{
   582  				&ValueExpr{
   583  					ValueExprMode: VEMStringExpr,
   584  					StringExpr: &StringExpr{
   585  						StringExprMode: SEMConcatExpr,
   586  						ConcatExpr: &ConcatExpr{
   587  							Atoms: []*ConcatAtom{
   588  								{IsField: false, Value: "Mary"},
   589  								{IsField: false, Value: "land"},
   590  							},
   591  						},
   592  					},
   593  				},
   594  				&ValueExpr{
   595  					ValueExprMode: VEMStringExpr,
   596  					StringExpr: &StringExpr{
   597  						StringExprMode: SEMRawString,
   598  						RawString:      "Hawaii",
   599  					},
   600  				},
   601  				&ValueExpr{
   602  					ValueExprMode: VEMNumericExpr,
   603  					NumericExpr: &NumericExpr{
   604  						NumericExprMode: NEMNumericExpr,
   605  						Op:              "+",
   606  						Left: &NumericExpr{
   607  							NumericExprMode: NEMNumber,
   608  							IsTerminal:      true,
   609  							ValueIsField:    false,
   610  							Value:           "99",
   611  						},
   612  						Right: &NumericExpr{
   613  							NumericExprMode: NEMNumber,
   614  							IsTerminal:      true,
   615  							ValueIsField:    false,
   616  							Value:           "1",
   617  						},
   618  					},
   619  				},
   620  			},
   621  		}
   622  
   623  	conditionExpr := &ConditionExpr{
   624  		Op:       "if",
   625  		BoolExpr: boolExpr,
   626  		TrueValue: &ValueExpr{
   627  			ValueExprMode: VEMNumericExpr,
   628  			NumericExpr: &NumericExpr{
   629  				NumericExprMode: NEMNumberField,
   630  				IsTerminal:      true,
   631  				ValueIsField:    true,
   632  				Value:           "state",
   633  			},
   634  		},
   635  		FalseValue: &ValueExpr{
   636  			ValueExprMode: VEMStringExpr,
   637  			StringExpr: &StringExpr{
   638  				StringExprMode: SEMRawString,
   639  				RawString:      "Error",
   640  			},
   641  		},
   642  	}
   643  
   644  	// Test GetFields()
   645  	assert.Equal(t, boolExpr.GetFields(), []string{"state"})
   646  
   647  	// Test Evaluate()
   648  	fieldToValue := make(map[string]segutils.CValueEnclosure)
   649  	fieldToValue["state"] = segutils.CValueEnclosure{
   650  		Dtype: segutils.SS_DT_STRING,
   651  		CVal:  "Maryland",
   652  	}
   653  
   654  	value, err := boolExpr.Evaluate(fieldToValue)
   655  	assert.Nil(t, err)
   656  	assert.Equal(t, value, true)
   657  
   658  	fieldToValue["state"] = segutils.CValueEnclosure{
   659  		Dtype: segutils.SS_DT_STRING,
   660  		CVal:  "MarylandTest",
   661  	}
   662  
   663  	value, err = boolExpr.Evaluate(fieldToValue)
   664  	assert.Nil(t, err)
   665  	assert.Equal(t, value, false)
   666  
   667  	fieldToValue["state"] = segutils.CValueEnclosure{
   668  		Dtype: segutils.SS_DT_STRING,
   669  		CVal:  "Hawaii",
   670  	}
   671  
   672  	//If true, it should return true value: state
   673  	str, err := conditionExpr.EvaluateCondition(fieldToValue)
   674  	assert.Nil(t, err)
   675  	assert.Equal(t, str, "Hawaii")
   676  
   677  	fieldToValue["state"] = segutils.CValueEnclosure{
   678  		Dtype: segutils.SS_DT_STRING,
   679  		CVal:  "NewYork",
   680  	}
   681  
   682  	//If false, it should return false value: "Error"
   683  	str, err = conditionExpr.EvaluateCondition(fieldToValue)
   684  	assert.Nil(t, err)
   685  	assert.Equal(t, str, "Error")
   686  
   687  	fieldToValue["state"] = segutils.CValueEnclosure{
   688  		Dtype: segutils.SS_DT_STRING,
   689  		CVal:  "Maryland",
   690  	}
   691  
   692  	str, err = conditionExpr.EvaluateCondition(fieldToValue)
   693  	assert.Nil(t, err)
   694  	assert.Equal(t, str, "Maryland")
   695  
   696  	boolExpr.LeftValue.NumericExpr.Value = "101"
   697  	boolExpr.LeftValue.NumericExpr.ValueIsField = false
   698  	boolExpr.LeftValue.NumericExpr.NumericExprMode = NEMNumber
   699  	value, err = boolExpr.Evaluate(fieldToValue)
   700  	assert.Nil(t, err)
   701  	assert.Equal(t, value, false)
   702  
   703  	boolExpr.LeftValue.NumericExpr.Value = "100"
   704  	value, err = boolExpr.Evaluate(fieldToValue)
   705  	assert.Nil(t, err)
   706  	assert.Equal(t, value, true)
   707  
   708  	isStr :=
   709  		&BoolExpr{
   710  			IsTerminal: true,
   711  			LeftValue: &ValueExpr{
   712  				ValueExprMode: VEMNumericExpr,
   713  				NumericExpr: &NumericExpr{
   714  					NumericExprMode: NEMNumberField,
   715  					IsTerminal:      true,
   716  					ValueIsField:    true,
   717  					Value:           "country",
   718  				},
   719  			},
   720  			ValueOp: "isstr",
   721  		}
   722  	isStrIf :=
   723  		&ConditionExpr{
   724  			Op:       "if",
   725  			BoolExpr: boolExpr,
   726  			TrueValue: &ValueExpr{
   727  				ValueExprMode: VEMStringExpr,
   728  				StringExpr: &StringExpr{
   729  					StringExprMode: SEMRawString,
   730  					RawString:      "This is a string",
   731  				},
   732  			},
   733  			FalseValue: &ValueExpr{
   734  				ValueExprMode: VEMStringExpr,
   735  				StringExpr: &StringExpr{
   736  					StringExprMode: SEMRawString,
   737  					RawString:      "This is not a string",
   738  				},
   739  			},
   740  		}
   741  
   742  	assert.Equal(t, isStr.GetFields(), []string{"country"})
   743  	fieldToValue["country"] = segutils.CValueEnclosure{
   744  		Dtype: segutils.SS_DT_STRING,
   745  		CVal:  "Spain",
   746  	}
   747  
   748  	str, err = isStrIf.EvaluateCondition(fieldToValue)
   749  	assert.Nil(t, err)
   750  	assert.Equal(t, str, "This is a string")
   751  
   752  	isNotStr :=
   753  		&BoolExpr{
   754  			IsTerminal: true,
   755  			LeftValue: &ValueExpr{
   756  				ValueExprMode: VEMNumericExpr,
   757  				NumericExpr: &NumericExpr{
   758  					NumericExprMode: NEMNumberField,
   759  					IsTerminal:      true,
   760  					ValueIsField:    true,
   761  					Value:           "longitude",
   762  				},
   763  			},
   764  			ValueOp: "isstr",
   765  		}
   766  	isNotStrIf :=
   767  		&ConditionExpr{
   768  			Op:       "if",
   769  			BoolExpr: isNotStr,
   770  			TrueValue: &ValueExpr{
   771  				ValueExprMode: VEMStringExpr,
   772  				StringExpr: &StringExpr{
   773  					StringExprMode: SEMRawString,
   774  					RawString:      "This is a string",
   775  				},
   776  			},
   777  			FalseValue: &ValueExpr{
   778  				ValueExprMode: VEMStringExpr,
   779  				StringExpr: &StringExpr{
   780  					StringExprMode: SEMRawString,
   781  					RawString:      "This is not a string",
   782  				},
   783  			},
   784  		}
   785  
   786  	assert.Equal(t, isNotStr.GetFields(), []string{"longitude"})
   787  	fieldToValue["longitude"] = segutils.CValueEnclosure{
   788  		Dtype: segutils.SS_DT_FLOAT,
   789  		CVal:  float64(99.619024),
   790  	}
   791  
   792  	str, err = isNotStrIf.EvaluateCondition(fieldToValue)
   793  	assert.Nil(t, err)
   794  	assert.Equal(t, str, "This is not a string")
   795  
   796  	isInt :=
   797  		&BoolExpr{
   798  			IsTerminal: true,
   799  			LeftValue: &ValueExpr{
   800  				ValueExprMode: VEMNumericExpr,
   801  				NumericExpr: &NumericExpr{
   802  					NumericExprMode: NEMNumberField,
   803  					IsTerminal:      true,
   804  					ValueIsField:    true,
   805  					Value:           "http_status",
   806  				},
   807  			},
   808  			ValueOp: "isint",
   809  		}
   810  	isIntIf :=
   811  		&ConditionExpr{
   812  			Op:       "if",
   813  			BoolExpr: isInt,
   814  			TrueValue: &ValueExpr{
   815  				ValueExprMode: VEMStringExpr,
   816  				StringExpr: &StringExpr{
   817  					StringExprMode: SEMRawString,
   818  					RawString:      "This is an integer",
   819  				},
   820  			},
   821  			FalseValue: &ValueExpr{
   822  				ValueExprMode: VEMStringExpr,
   823  				StringExpr: &StringExpr{
   824  					StringExprMode: SEMRawString,
   825  					RawString:      "This is not an integer",
   826  				},
   827  			},
   828  		}
   829  
   830  	assert.Equal(t, isInt.GetFields(), []string{"http_status"})
   831  	fieldToValue["http_status"] = segutils.CValueEnclosure{
   832  		Dtype: segutils.SS_DT_SIGNED_NUM,
   833  		CVal:  int64(500),
   834  	}
   835  
   836  	str, err = isIntIf.EvaluateCondition(fieldToValue)
   837  	assert.Nil(t, err)
   838  	assert.Equal(t, str, "This is an integer")
   839  
   840  	isNotInt :=
   841  		&BoolExpr{
   842  			IsTerminal: true,
   843  			LeftValue: &ValueExpr{
   844  				ValueExprMode: VEMNumericExpr,
   845  				NumericExpr: &NumericExpr{
   846  					NumericExprMode: NEMNumberField,
   847  					IsTerminal:      true,
   848  					ValueIsField:    true,
   849  					Value:           "longitude",
   850  				},
   851  			},
   852  			ValueOp: "isint",
   853  		}
   854  	isNotIntIf :=
   855  		&ConditionExpr{
   856  			Op:       "if",
   857  			BoolExpr: isNotInt,
   858  			TrueValue: &ValueExpr{
   859  				ValueExprMode: VEMStringExpr,
   860  				StringExpr: &StringExpr{
   861  					StringExprMode: SEMRawString,
   862  					RawString:      "This is an integer",
   863  				},
   864  			},
   865  			FalseValue: &ValueExpr{
   866  				ValueExprMode: VEMStringExpr,
   867  				StringExpr: &StringExpr{
   868  					StringExprMode: SEMRawString,
   869  					RawString:      "This is not an integer",
   870  				},
   871  			},
   872  		}
   873  
   874  	assert.Equal(t, isNotInt.GetFields(), []string{"longitude"})
   875  	fieldToValue["longitude"] = segutils.CValueEnclosure{
   876  		Dtype: segutils.SS_DT_FLOAT,
   877  		CVal:  float64(99.619024),
   878  	}
   879  
   880  	str, err = isNotIntIf.EvaluateCondition(fieldToValue)
   881  	assert.Nil(t, err)
   882  	assert.Equal(t, str, "This is not an integer")
   883  
   884  	isNotBool :=
   885  		&BoolExpr{
   886  			IsTerminal: true,
   887  			LeftValue: &ValueExpr{
   888  				ValueExprMode: VEMNumericExpr,
   889  				NumericExpr: &NumericExpr{
   890  					NumericExprMode: NEMNumberField,
   891  					IsTerminal:      true,
   892  					ValueIsField:    true,
   893  					Value:           "city",
   894  				},
   895  			},
   896  			ValueOp: "isbool",
   897  		}
   898  	isNotBoolIf :=
   899  		&ConditionExpr{
   900  			Op:       "if",
   901  			BoolExpr: isNotBool,
   902  			TrueValue: &ValueExpr{
   903  				ValueExprMode: VEMStringExpr,
   904  				StringExpr: &StringExpr{
   905  					StringExprMode: SEMRawString,
   906  					RawString:      "This is a boolean value",
   907  				},
   908  			},
   909  			FalseValue: &ValueExpr{
   910  				ValueExprMode: VEMStringExpr,
   911  				StringExpr: &StringExpr{
   912  					StringExprMode: SEMRawString,
   913  					RawString:      "This is not a boolean value",
   914  				},
   915  			},
   916  		}
   917  
   918  	assert.Equal(t, isNotBool.GetFields(), []string{"city"})
   919  	fieldToValue["city"] = segutils.CValueEnclosure{
   920  		Dtype: segutils.SS_DT_STRING,
   921  		CVal:  "Boston",
   922  	}
   923  
   924  	str, err = isNotBoolIf.EvaluateCondition(fieldToValue)
   925  	assert.Nil(t, err)
   926  	assert.Equal(t, str, "This is not a boolean value")
   927  
   928  	isBool :=
   929  		&BoolExpr{
   930  			IsTerminal: true,
   931  			LeftValue: &ValueExpr{
   932  				ValueExprMode: VEMNumericExpr,
   933  				NumericExpr: &NumericExpr{
   934  					NumericExprMode: NEMNumber,
   935  					IsTerminal:      true,
   936  					ValueIsField:    false,
   937  					Value:           "true",
   938  				},
   939  			},
   940  			ValueOp: "isbool",
   941  		}
   942  	isBoolIf :=
   943  		&ConditionExpr{
   944  			Op:       "if",
   945  			BoolExpr: isBool,
   946  			TrueValue: &ValueExpr{
   947  				ValueExprMode: VEMStringExpr,
   948  				StringExpr: &StringExpr{
   949  					StringExprMode: SEMRawString,
   950  					RawString:      "This is a boolean value",
   951  				},
   952  			},
   953  			FalseValue: &ValueExpr{
   954  				ValueExprMode: VEMStringExpr,
   955  				StringExpr: &StringExpr{
   956  					StringExprMode: SEMRawString,
   957  					RawString:      "This is not a boolean value",
   958  				},
   959  			},
   960  		}
   961  
   962  	assert.Equal(t, isBool.GetFields(), []string{})
   963  	fieldToValue["true"] = segutils.CValueEnclosure{
   964  		Dtype: segutils.SS_DT_STRING,
   965  		CVal:  "true",
   966  	}
   967  
   968  	str, err = isBoolIf.EvaluateCondition(fieldToValue)
   969  	assert.Nil(t, err)
   970  	assert.Equal(t, str, "This is a boolean value")
   971  
   972  	IsNotNull :=
   973  		&BoolExpr{
   974  			IsTerminal: true,
   975  			LeftValue: &ValueExpr{
   976  				ValueExprMode: VEMNumericExpr,
   977  				NumericExpr: &NumericExpr{
   978  					NumericExprMode: NEMNumberField,
   979  					IsTerminal:      true,
   980  					ValueIsField:    true,
   981  					Value:           "state",
   982  				},
   983  			},
   984  			ValueOp: "isbool",
   985  		}
   986  	isNotNullIf :=
   987  		&ConditionExpr{
   988  			Op:       "if",
   989  			BoolExpr: IsNotNull,
   990  			TrueValue: &ValueExpr{
   991  				ValueExprMode: VEMStringExpr,
   992  				StringExpr: &StringExpr{
   993  					StringExprMode: SEMRawString,
   994  					RawString:      "This is a null value",
   995  				},
   996  			},
   997  			FalseValue: &ValueExpr{
   998  				ValueExprMode: VEMStringExpr,
   999  				StringExpr: &StringExpr{
  1000  					StringExprMode: SEMRawString,
  1001  					RawString:      "This is not a null value",
  1002  				},
  1003  			},
  1004  		}
  1005  
  1006  	assert.Equal(t, IsNotNull.GetFields(), []string{"state"})
  1007  	fieldToValue["state"] = segutils.CValueEnclosure{
  1008  		Dtype: segutils.SS_DT_STRING,
  1009  		CVal:  "Florida",
  1010  	}
  1011  
  1012  	str, err = isNotNullIf.EvaluateCondition(fieldToValue)
  1013  	assert.Nil(t, err)
  1014  	assert.Equal(t, str, "This is not a null value")
  1015  
  1016  	cidrBoolExpr :=
  1017  		&BoolExpr{
  1018  			IsTerminal: true,
  1019  			LeftValue: &ValueExpr{
  1020  				ValueExprMode: VEMStringExpr,
  1021  				StringExpr: &StringExpr{
  1022  					StringExprMode: SEMRawString,
  1023  					RawString:      "192.0.2.0/24",
  1024  				},
  1025  			},
  1026  			RightValue: &ValueExpr{
  1027  				ValueExprMode: VEMStringExpr,
  1028  				StringExpr: &StringExpr{
  1029  					StringExprMode: SEMRawString,
  1030  					RawString:      "192.0.2.5",
  1031  				},
  1032  			},
  1033  			ValueOp: "cidrmatch",
  1034  		}
  1035  	cidrConditionExpr :=
  1036  		&ConditionExpr{
  1037  			Op:       "if",
  1038  			BoolExpr: cidrBoolExpr,
  1039  			TrueValue: &ValueExpr{
  1040  				ValueExprMode: VEMStringExpr,
  1041  				StringExpr: &StringExpr{
  1042  					StringExprMode: SEMRawString,
  1043  					RawString:      "local",
  1044  				},
  1045  			},
  1046  			FalseValue: &ValueExpr{
  1047  				ValueExprMode: VEMStringExpr,
  1048  				StringExpr: &StringExpr{
  1049  					StringExprMode: SEMRawString,
  1050  					RawString:      "not local",
  1051  				},
  1052  			},
  1053  		}
  1054  
  1055  	assert.Equal(t, cidrBoolExpr.GetFields(), []string{})
  1056  
  1057  	str, err = cidrConditionExpr.EvaluateCondition(fieldToValue)
  1058  	assert.Nil(t, err)
  1059  	assert.Equal(t, str, "local")
  1060  
  1061  	notCidrMatchBoolExpr :=
  1062  		&BoolExpr{
  1063  			IsTerminal: true,
  1064  			LeftValue: &ValueExpr{
  1065  				ValueExprMode: VEMStringExpr,
  1066  				StringExpr: &StringExpr{
  1067  					StringExprMode: SEMRawString,
  1068  					RawString:      "192.0.2.0/24",
  1069  				},
  1070  			},
  1071  			RightValue: &ValueExpr{
  1072  				ValueExprMode: VEMStringExpr,
  1073  				StringExpr: &StringExpr{
  1074  					StringExprMode: SEMRawString,
  1075  					RawString:      "192.0.3.1",
  1076  				},
  1077  			},
  1078  			ValueOp: "cidrmatch",
  1079  		}
  1080  	notCidrMatchConditionExpr :=
  1081  		&ConditionExpr{
  1082  			Op:       "if",
  1083  			BoolExpr: notCidrMatchBoolExpr,
  1084  			TrueValue: &ValueExpr{
  1085  				ValueExprMode: VEMStringExpr,
  1086  				StringExpr: &StringExpr{
  1087  					StringExprMode: SEMRawString,
  1088  					RawString:      "local",
  1089  				},
  1090  			},
  1091  			FalseValue: &ValueExpr{
  1092  				ValueExprMode: VEMStringExpr,
  1093  				StringExpr: &StringExpr{
  1094  					StringExprMode: SEMRawString,
  1095  					RawString:      "not local",
  1096  				},
  1097  			},
  1098  		}
  1099  	assert.Equal(t, notCidrMatchBoolExpr.GetFields(), []string{})
  1100  
  1101  	str, err = notCidrMatchConditionExpr.EvaluateCondition(fieldToValue)
  1102  	assert.Nil(t, err)
  1103  	assert.Equal(t, str, "not local")
  1104  
  1105  	likeBoolExpr :=
  1106  		&BoolExpr{
  1107  			IsTerminal: true,
  1108  			LeftValue: &ValueExpr{
  1109  				ValueExprMode: VEMNumericExpr,
  1110  				NumericExpr: &NumericExpr{
  1111  					NumericExprMode: NEMNumberField,
  1112  					IsTerminal:      true,
  1113  					ValueIsField:    true,
  1114  					Value:           "http_status",
  1115  				},
  1116  			},
  1117  			RightValue: &ValueExpr{
  1118  				ValueExprMode: VEMStringExpr,
  1119  				StringExpr: &StringExpr{
  1120  					StringExprMode: SEMRawString,
  1121  					RawString:      "4%",
  1122  				},
  1123  			},
  1124  			ValueOp: "like",
  1125  		}
  1126  	likeConditionExpr :=
  1127  		&ConditionExpr{
  1128  			Op:       "if",
  1129  			BoolExpr: likeBoolExpr,
  1130  			TrueValue: &ValueExpr{
  1131  				ValueExprMode: VEMStringExpr,
  1132  				StringExpr: &StringExpr{
  1133  					StringExprMode: SEMRawString,
  1134  					RawString:      "True",
  1135  				},
  1136  			},
  1137  			FalseValue: &ValueExpr{
  1138  				ValueExprMode: VEMStringExpr,
  1139  				StringExpr: &StringExpr{
  1140  					StringExprMode: SEMRawString,
  1141  					RawString:      "False",
  1142  				},
  1143  			},
  1144  		}
  1145  
  1146  	assert.Equal(t, likeBoolExpr.GetFields(), []string{"http_status"})
  1147  	fieldToValue["http_status"] = segutils.CValueEnclosure{
  1148  		Dtype: segutils.SS_DT_SIGNED_NUM,
  1149  		CVal:  int64(400),
  1150  	}
  1151  
  1152  	str, err = likeConditionExpr.EvaluateCondition(fieldToValue)
  1153  	assert.Nil(t, err)
  1154  	assert.Equal(t, str, "True")
  1155  
  1156  	notLikeBoolExpr :=
  1157  		&BoolExpr{
  1158  			IsTerminal: true,
  1159  			LeftValue: &ValueExpr{
  1160  				ValueExprMode: VEMNumericExpr,
  1161  				NumericExpr: &NumericExpr{
  1162  					NumericExprMode: NEMNumberField,
  1163  					IsTerminal:      true,
  1164  					ValueIsField:    true,
  1165  					Value:           "http_status",
  1166  				},
  1167  			},
  1168  			RightValue: &ValueExpr{
  1169  				ValueExprMode: VEMStringExpr,
  1170  				StringExpr: &StringExpr{
  1171  					StringExprMode: SEMRawString,
  1172  					RawString:      "4%",
  1173  				},
  1174  			},
  1175  			ValueOp: "like",
  1176  		}
  1177  	notLikeConditionExpr :=
  1178  		&ConditionExpr{
  1179  			Op:       "if",
  1180  			BoolExpr: likeBoolExpr,
  1181  			TrueValue: &ValueExpr{
  1182  				ValueExprMode: VEMStringExpr,
  1183  				StringExpr: &StringExpr{
  1184  					StringExprMode: SEMRawString,
  1185  					RawString:      "True",
  1186  				},
  1187  			},
  1188  			FalseValue: &ValueExpr{
  1189  				ValueExprMode: VEMStringExpr,
  1190  				StringExpr: &StringExpr{
  1191  					StringExprMode: SEMRawString,
  1192  					RawString:      "False",
  1193  				},
  1194  			},
  1195  		}
  1196  
  1197  	assert.Equal(t, notLikeBoolExpr.GetFields(), []string{"http_status"})
  1198  	fieldToValue["http_status"] = segutils.CValueEnclosure{
  1199  		Dtype: segutils.SS_DT_SIGNED_NUM,
  1200  		CVal:  int64(200),
  1201  	}
  1202  
  1203  	str, err = notLikeConditionExpr.EvaluateCondition(fieldToValue)
  1204  	assert.Nil(t, err)
  1205  	assert.Equal(t, str, "False")
  1206  
  1207  	matchBoolExpr :=
  1208  		&BoolExpr{
  1209  			IsTerminal: true,
  1210  			LeftValue: &ValueExpr{
  1211  				ValueExprMode: VEMNumericExpr,
  1212  				NumericExpr: &NumericExpr{
  1213  					NumericExprMode: NEMNumberField,
  1214  					IsTerminal:      true,
  1215  					ValueIsField:    true,
  1216  					Value:           "country",
  1217  				},
  1218  			},
  1219  			RightValue: &ValueExpr{
  1220  				ValueExprMode: VEMStringExpr,
  1221  				StringExpr: &StringExpr{
  1222  					StringExprMode: SEMRawString,
  1223  					RawString:      "^Sa",
  1224  				},
  1225  			},
  1226  			ValueOp: "match",
  1227  		}
  1228  	matchConditionExpr :=
  1229  		&ConditionExpr{
  1230  			Op:       "if",
  1231  			BoolExpr: matchBoolExpr,
  1232  			TrueValue: &ValueExpr{
  1233  				ValueExprMode: VEMStringExpr,
  1234  				StringExpr: &StringExpr{
  1235  					StringExprMode: SEMRawString,
  1236  					RawString:      "yes",
  1237  				},
  1238  			},
  1239  			FalseValue: &ValueExpr{
  1240  				ValueExprMode: VEMStringExpr,
  1241  				StringExpr: &StringExpr{
  1242  					StringExprMode: SEMRawString,
  1243  					RawString:      "no",
  1244  				},
  1245  			},
  1246  		}
  1247  
  1248  	assert.Equal(t, matchBoolExpr.GetFields(), []string{"country"})
  1249  	fieldToValue["country"] = segutils.CValueEnclosure{
  1250  		Dtype: segutils.SS_DT_STRING,
  1251  		CVal:  "Saudi Arabia",
  1252  	}
  1253  
  1254  	str, err = matchConditionExpr.EvaluateCondition(fieldToValue)
  1255  	assert.Nil(t, err)
  1256  	assert.Equal(t, str, "yes")
  1257  
  1258  	notMatchBoolExpr :=
  1259  		&BoolExpr{
  1260  			IsTerminal: true,
  1261  			LeftValue: &ValueExpr{
  1262  				ValueExprMode: VEMNumericExpr,
  1263  				NumericExpr: &NumericExpr{
  1264  					NumericExprMode: NEMNumberField,
  1265  					IsTerminal:      true,
  1266  					ValueIsField:    true,
  1267  					Value:           "country",
  1268  				},
  1269  			},
  1270  			RightValue: &ValueExpr{
  1271  				ValueExprMode: VEMStringExpr,
  1272  				StringExpr: &StringExpr{
  1273  					StringExprMode: SEMRawString,
  1274  					RawString:      "^Sa",
  1275  				},
  1276  			},
  1277  			ValueOp: "match",
  1278  		}
  1279  	notMatchConditionExpr :=
  1280  		&ConditionExpr{
  1281  			Op:       "if",
  1282  			BoolExpr: notMatchBoolExpr,
  1283  			TrueValue: &ValueExpr{
  1284  				ValueExprMode: VEMStringExpr,
  1285  				StringExpr: &StringExpr{
  1286  					StringExprMode: SEMRawString,
  1287  					RawString:      "yes",
  1288  				},
  1289  			},
  1290  			FalseValue: &ValueExpr{
  1291  				ValueExprMode: VEMStringExpr,
  1292  				StringExpr: &StringExpr{
  1293  					StringExprMode: SEMRawString,
  1294  					RawString:      "no",
  1295  				},
  1296  			},
  1297  		}
  1298  
  1299  	assert.Equal(t, notMatchBoolExpr.GetFields(), []string{"country"})
  1300  	fieldToValue["country"] = segutils.CValueEnclosure{
  1301  		Dtype: segutils.SS_DT_STRING,
  1302  		CVal:  "Jersey",
  1303  	}
  1304  
  1305  	str, err = notMatchConditionExpr.EvaluateCondition(fieldToValue)
  1306  	assert.Nil(t, err)
  1307  	assert.Equal(t, str, "no")
  1308  
  1309  }
  1310  
  1311  func Test_StringExpr(t *testing.T) {
  1312  	strExpr :=
  1313  		&StringExpr{
  1314  			StringExprMode: SEMTextExpr,
  1315  			TextExpr: &TextExpr{
  1316  				IsTerminal: false,
  1317  				Op:         "urldecode",
  1318  				Value: &StringExpr{
  1319  					StringExprMode: SEMRawString,
  1320  					RawString:      "http%3A%2F%2Fwww.splunk.com%2Fdownload%3Fr%3Dheader",
  1321  				},
  1322  			},
  1323  		}
  1324  	assert.Equal(t, strExpr.GetFields(), []string{})
  1325  
  1326  	// Test Evaluate()
  1327  	fieldToValue := make(map[string]segutils.CValueEnclosure)
  1328  
  1329  	value, err := strExpr.Evaluate(fieldToValue)
  1330  	assert.Nil(t, err)
  1331  	assert.Equal(t, value, "http://www.splunk.com/download?r=header")
  1332  
  1333  	strExpr1 :=
  1334  		&StringExpr{
  1335  			StringExprMode: SEMTextExpr,
  1336  			TextExpr: &TextExpr{
  1337  				IsTerminal: false,
  1338  				Op:         "split",
  1339  				Value: &StringExpr{
  1340  					StringExprMode: SEMField,
  1341  					FieldName:      "ident",
  1342  				},
  1343  				Delimiter: &StringExpr{
  1344  					StringExprMode: SEMRawString,
  1345  					RawString:      "-",
  1346  				},
  1347  			},
  1348  		}
  1349  	assert.Equal(t, strExpr1.GetFields(), []string{"ident"})
  1350  
  1351  	// Test Evaluate()
  1352  	fieldToValue["ident"] = segutils.CValueEnclosure{
  1353  		Dtype: segutils.SS_DT_STRING,
  1354  		CVal:  "a111d29d-dd70-48b2-8987-a807b4b8bbae",
  1355  	}
  1356  
  1357  	value, err = strExpr1.Evaluate(fieldToValue)
  1358  	assert.Nil(t, err)
  1359  	assert.Equal(t, value, "a111d29d&nbspdd70&nbsp48b2&nbsp8987&nbspa807b4b8bbae")
  1360  
  1361  	strMax :=
  1362  		&StringExpr{
  1363  			StringExprMode: SEMTextExpr,
  1364  			TextExpr: &TextExpr{
  1365  				IsTerminal: false,
  1366  				Op:         "max",
  1367  				MaxMinValues: []*StringExpr{
  1368  					{
  1369  						StringExprMode: SEMConcatExpr,
  1370  						ConcatExpr: &ConcatExpr{
  1371  							Atoms: []*ConcatAtom{
  1372  								{IsField: false, Value: "1"},
  1373  							},
  1374  						},
  1375  					},
  1376  
  1377  					{
  1378  						StringExprMode: SEMConcatExpr,
  1379  						ConcatExpr: &ConcatExpr{
  1380  							Atoms: []*ConcatAtom{
  1381  								{IsField: false, Value: "3"},
  1382  							},
  1383  						},
  1384  					},
  1385  
  1386  					{
  1387  						StringExprMode: SEMConcatExpr,
  1388  						ConcatExpr: &ConcatExpr{
  1389  							Atoms: []*ConcatAtom{
  1390  								{IsField: false, Value: "450"},
  1391  							},
  1392  						},
  1393  					},
  1394  					{
  1395  
  1396  						StringExprMode: SEMField,
  1397  						FieldName:      "http_status",
  1398  					},
  1399  				},
  1400  			},
  1401  		}
  1402  	assert.Equal(t, strMax.GetFields(), []string{"http_status"})
  1403  
  1404  	// Test Evaluate()
  1405  	fieldToValue["http_status"] = segutils.CValueEnclosure{
  1406  		Dtype: segutils.SS_DT_STRING,
  1407  		CVal:  "200",
  1408  	}
  1409  
  1410  	value, err = strMax.Evaluate(fieldToValue)
  1411  	assert.Nil(t, err)
  1412  	assert.Equal(t, value, "450")
  1413  
  1414  	strMin :=
  1415  		&StringExpr{
  1416  			StringExprMode: SEMTextExpr,
  1417  			TextExpr: &TextExpr{
  1418  				IsTerminal: false,
  1419  				Op:         "min",
  1420  				MaxMinValues: []*StringExpr{
  1421  					{
  1422  						StringExprMode: SEMConcatExpr,
  1423  						ConcatExpr: &ConcatExpr{
  1424  							Atoms: []*ConcatAtom{
  1425  								{IsField: false, Value: "1"},
  1426  							},
  1427  						},
  1428  					},
  1429  
  1430  					{
  1431  						StringExprMode: SEMConcatExpr,
  1432  						ConcatExpr: &ConcatExpr{
  1433  							Atoms: []*ConcatAtom{
  1434  								{IsField: false, Value: "3"},
  1435  							},
  1436  						},
  1437  					},
  1438  
  1439  					{
  1440  						StringExprMode: SEMConcatExpr,
  1441  						ConcatExpr: &ConcatExpr{
  1442  							Atoms: []*ConcatAtom{
  1443  								{IsField: false, Value: "450"},
  1444  							},
  1445  						},
  1446  					},
  1447  					{
  1448  
  1449  						StringExprMode: SEMField,
  1450  						FieldName:      "http_status",
  1451  					},
  1452  				},
  1453  			},
  1454  		}
  1455  	assert.Equal(t, strMin.GetFields(), []string{"http_status"})
  1456  
  1457  	// Test Evaluate()
  1458  	fieldToValue["http_status"] = segutils.CValueEnclosure{
  1459  		Dtype: segutils.SS_DT_STRING,
  1460  		CVal:  "200",
  1461  	}
  1462  
  1463  	value, err = strMin.Evaluate(fieldToValue)
  1464  	assert.Nil(t, err)
  1465  	assert.Equal(t, value, "1")
  1466  
  1467  	strSubStr := &StringExpr{
  1468  		StringExprMode: SEMConcatExpr,
  1469  		ConcatExpr: &ConcatExpr{
  1470  			Atoms: []*ConcatAtom{
  1471  				{
  1472  					IsField: false,
  1473  					TextExpr: &TextExpr{
  1474  						IsTerminal: false,
  1475  						Op:         "substr",
  1476  						Value: &StringExpr{
  1477  							StringExprMode: SEMRawString,
  1478  							RawString:      "splendid",
  1479  						},
  1480  						StartIndex: &NumericExpr{
  1481  							NumericExprMode: NEMNumber,
  1482  							IsTerminal:      true,
  1483  							ValueIsField:    false,
  1484  							Value:           "1",
  1485  						},
  1486  						LengthExpr: &NumericExpr{
  1487  							NumericExprMode: NEMNumber,
  1488  							IsTerminal:      true,
  1489  							ValueIsField:    false,
  1490  							Value:           "3",
  1491  						},
  1492  					},
  1493  				},
  1494  				{
  1495  					IsField: false,
  1496  					TextExpr: &TextExpr{
  1497  						IsTerminal: false,
  1498  						Op:         "substr",
  1499  						Value: &StringExpr{
  1500  							StringExprMode: SEMRawString,
  1501  							RawString:      "chunk",
  1502  						},
  1503  						StartIndex: &NumericExpr{
  1504  							NumericExprMode: NEMNumber,
  1505  							IsTerminal:      true,
  1506  							ValueIsField:    false,
  1507  							Value:           "-3",
  1508  						},
  1509  					},
  1510  				},
  1511  			},
  1512  		},
  1513  	}
  1514  	assert.Equal(t, strSubStr.GetFields(), []string{})
  1515  
  1516  	value, err = strSubStr.Evaluate(fieldToValue)
  1517  	assert.Nil(t, err)
  1518  	assert.Equal(t, value, "splunk")
  1519  
  1520  	strToStringBool :=
  1521  		&StringExpr{
  1522  			StringExprMode: SEMTextExpr,
  1523  			TextExpr: &TextExpr{
  1524  				IsTerminal: false,
  1525  				Op:         "tostring",
  1526  				Val: &ValueExpr{
  1527  					ValueExprMode: VEMBooleanExpr,
  1528  					BooleanExpr: &BoolExpr{
  1529  						IsTerminal: true,
  1530  						LeftValue: &ValueExpr{
  1531  							ValueExprMode: VEMNumericExpr,
  1532  							NumericExpr: &NumericExpr{
  1533  								NumericExprMode: NEMNumber,
  1534  								IsTerminal:      true,
  1535  								ValueIsField:    false,
  1536  								Value:           "2",
  1537  							},
  1538  						},
  1539  						RightValue: &ValueExpr{
  1540  							ValueExprMode: VEMNumericExpr,
  1541  							NumericExpr: &NumericExpr{
  1542  								NumericExprMode: NEMNumber,
  1543  								IsTerminal:      true,
  1544  								ValueIsField:    false,
  1545  								Value:           "1",
  1546  							},
  1547  						},
  1548  						ValueOp: ">",
  1549  					},
  1550  				},
  1551  			},
  1552  		}
  1553  	assert.Equal(t, strToStringBool.GetFields(), []string{})
  1554  
  1555  	value, err = strToStringBool.Evaluate(fieldToValue)
  1556  	assert.Nil(t, err)
  1557  	assert.Equal(t, value, "true")
  1558  
  1559  	strToStringHex :=
  1560  		&StringExpr{
  1561  			StringExprMode: SEMTextExpr,
  1562  			TextExpr: &TextExpr{
  1563  				IsTerminal: false,
  1564  				Op:         "tostring",
  1565  				Val: &ValueExpr{
  1566  					ValueExprMode: VEMNumericExpr,
  1567  					NumericExpr: &NumericExpr{
  1568  						NumericExprMode: NEMNumber,
  1569  						IsTerminal:      true,
  1570  						ValueIsField:    false,
  1571  						Value:           "15",
  1572  					},
  1573  				},
  1574  				Format: &StringExpr{
  1575  					StringExprMode: SEMRawString,
  1576  					RawString:      "hex",
  1577  				},
  1578  			},
  1579  		}
  1580  	assert.Equal(t, strToStringHex.GetFields(), []string{})
  1581  
  1582  	value, err = strToStringHex.Evaluate(fieldToValue)
  1583  	assert.Nil(t, err)
  1584  	assert.Equal(t, value, "0xf")
  1585  
  1586  	strToStringCommas :=
  1587  		&StringExpr{
  1588  			StringExprMode: SEMTextExpr,
  1589  			TextExpr: &TextExpr{
  1590  				IsTerminal: false,
  1591  				Op:         "tostring",
  1592  				Val: &ValueExpr{
  1593  					ValueExprMode: VEMNumericExpr,
  1594  					NumericExpr: &NumericExpr{
  1595  						NumericExprMode: NEMNumber,
  1596  						IsTerminal:      true,
  1597  						ValueIsField:    false,
  1598  						Value:           "12345.6789",
  1599  					},
  1600  				},
  1601  				Format: &StringExpr{
  1602  					StringExprMode: SEMRawString,
  1603  					RawString:      "commas",
  1604  				},
  1605  			},
  1606  		}
  1607  	assert.Equal(t, strToStringCommas.GetFields(), []string{})
  1608  
  1609  	value, err = strToStringCommas.Evaluate(fieldToValue)
  1610  	assert.Nil(t, err)
  1611  	assert.Equal(t, value, "12,345.68")
  1612  
  1613  	strToStringDuration :=
  1614  		&StringExpr{
  1615  			StringExprMode: SEMTextExpr,
  1616  			TextExpr: &TextExpr{
  1617  				IsTerminal: false,
  1618  				Op:         "tostring",
  1619  				Val: &ValueExpr{
  1620  					ValueExprMode: VEMNumericExpr,
  1621  					NumericExpr: &NumericExpr{
  1622  						NumericExprMode: NEMNumber,
  1623  						IsTerminal:      true,
  1624  						ValueIsField:    false,
  1625  						Value:           "615",
  1626  					},
  1627  				},
  1628  				Format: &StringExpr{
  1629  					StringExprMode: SEMRawString,
  1630  					RawString:      "duration",
  1631  				},
  1632  			},
  1633  		}
  1634  	assert.Equal(t, strToStringDuration.GetFields(), []string{})
  1635  
  1636  	value, err = strToStringDuration.Evaluate(fieldToValue)
  1637  	assert.Nil(t, err)
  1638  	assert.Equal(t, value, "00:10:15")
  1639  
  1640  }
  1641  
  1642  func Test_RenameExpr(t *testing.T) {
  1643  	renameToPhrase := &RenameExpr{
  1644  		RenameExprMode:  REMPhrase,
  1645  		OriginalPattern: "city",
  1646  		NewPattern:      "test",
  1647  	}
  1648  
  1649  	renameRegex := &RenameExpr{
  1650  		RenameExprMode:  REMRegex,
  1651  		OriginalPattern: "app*",
  1652  		NewPattern:      "start*end",
  1653  	}
  1654  
  1655  	renameToExistingField := &RenameExpr{
  1656  		RenameExprMode:  REMOverride,
  1657  		OriginalPattern: "http_status",
  1658  		NewPattern:      "",
  1659  	}
  1660  
  1661  	assert.Equal(t, []string{"city"}, renameToPhrase.GetFields())
  1662  	assert.Equal(t, []string{}, renameRegex.GetFields())
  1663  	assert.Equal(t, []string{"http_status"}, renameToExistingField.GetFields())
  1664  
  1665  	fieldToValue := make(map[string]segutils.CValueEnclosure)
  1666  	fieldToValue["city"] = segutils.CValueEnclosure{
  1667  		Dtype: segutils.SS_DT_STRING,
  1668  		CVal:  "Boston",
  1669  	}
  1670  	fieldToValue["http_status"] = segutils.CValueEnclosure{
  1671  		Dtype: segutils.SS_DT_STRING,
  1672  		CVal:  "200",
  1673  	}
  1674  
  1675  	val, err := renameToPhrase.Evaluate(fieldToValue, renameToPhrase.GetFields()[0])
  1676  	assert.Nil(t, err)
  1677  	assert.Equal(t, "Boston", val)
  1678  
  1679  	val, err = renameToExistingField.Evaluate(fieldToValue, renameToExistingField.GetFields()[0])
  1680  	assert.Nil(t, err)
  1681  	assert.Equal(t, "200", val)
  1682  
  1683  	// Test Process Rename Regex logic
  1684  	// No match column
  1685  	newCol, err := renameRegex.ProcessRenameRegexExpression("http_status")
  1686  	assert.Nil(t, err)
  1687  	assert.Equal(t, "", newCol)
  1688  
  1689  	newCol, err = renameRegex.ProcessRenameRegexExpression("app_name")
  1690  	assert.Nil(t, err)
  1691  	assert.Equal(t, "start_nameend", newCol)
  1692  
  1693  	// Multiple wildcards
  1694  	renameRegex.OriginalPattern = "ht*_*ta*"
  1695  	renameRegex.NewPattern = "first*second*third*end"
  1696  	newCol, err = renameRegex.ProcessRenameRegexExpression("http_status")
  1697  
  1698  	assert.Nil(t, err)
  1699  	assert.Equal(t, newCol, "firsttpsecondsthirdtusend")
  1700  
  1701  	// Wrong Pattern
  1702  	renameRegex.OriginalPattern = "[abc"
  1703  	renameRegex.NewPattern = "first*second*third*end"
  1704  	_, err = renameRegex.ProcessRenameRegexExpression("ddd")
  1705  
  1706  	assert.NotNil(t, err)
  1707  
  1708  	// Test Remove unused GroupByCols by index
  1709  	bucketResult := &BucketResult{
  1710  		GroupByKeys: []string{"http_status", "http_method", "city", "state", "gender", "app_name"},
  1711  		BucketKey:   []string{"200", "POST", "Boston", "MA", "Male", "sig"},
  1712  	}
  1713  
  1714  	renameRegex.RemoveBucketResGroupByColumnsByIndex(bucketResult, []int{3, 1, 4})
  1715  	assert.Equal(t, []string{"http_status", "city", "app_name"}, bucketResult.GroupByKeys)
  1716  	assert.Equal(t, []string{"200", "Boston", "sig"}, bucketResult.BucketKey.([]string))
  1717  
  1718  	bucketHolder := &BucketHolder{
  1719  		GroupByValues: []string{"200", "POST", "Boston", "MA", "Male", "sig"},
  1720  	}
  1721  
  1722  	renameRegex.RemoveBucketHolderGroupByColumnsByIndex(bucketHolder, []string{"http_status", "http_method", "city", "state", "gender", "app_name"}, []int{5, 2})
  1723  	assert.Equal(t, []string{"200", "POST", "MA", "Male"}, bucketHolder.GroupByValues)
  1724  }
  1725  
  1726  func Test_StatisticExpr(t *testing.T) {
  1727  
  1728  	statisticExpr := &StatisticExpr{
  1729  		StatisticFunctionMode: SFMRare,
  1730  		Limit:                 "2",
  1731  		StatisticOptions: &StatisticOptions{
  1732  			CountField:   "app_name",
  1733  			OtherStr:     "other",
  1734  			PercentField: "http_method",
  1735  			ShowCount:    true,
  1736  			ShowPerc:     true,
  1737  			UseOther:     true,
  1738  		},
  1739  		FieldList: []string{"http_method", "weekday"},
  1740  		ByClause:  []string{"app_name"},
  1741  	}
  1742  
  1743  	assert.Equal(t, []string{"http_method", "weekday", "app_name"}, statisticExpr.GetGroupByCols())
  1744  
  1745  	bucketResult := &BucketResult{
  1746  		ElemCount:   333,
  1747  		GroupByKeys: []string{"http_method", "weekday", "app_name"},
  1748  		BucketKey:   []string{"PUT", "Sunday", "sig"},
  1749  	}
  1750  
  1751  	err := statisticExpr.OverrideGroupByCol(bucketResult, 666)
  1752  	assert.Nil(t, err)
  1753  	assert.Equal(t, []string{"50.000000", "Sunday", "333"}, bucketResult.BucketKey.([]string))
  1754  
  1755  	bucketResult1 := &BucketResult{
  1756  		ElemCount:   333,
  1757  		GroupByKeys: []string{"http_method", "http_status", "weekday", "state", "gender", "app_name"},
  1758  		BucketKey:   []string{"POST", "404", "Sunday", "MA", "Male", "sig"},
  1759  	}
  1760  
  1761  	bucketResult2 := &BucketResult{
  1762  		ElemCount:   111,
  1763  		GroupByKeys: []string{"http_method", "http_status", "weekday", "state", "gender", "app_name"},
  1764  		BucketKey:   []string{"Get", "200", "Tuesday", "LA", "Male", "test"},
  1765  	}
  1766  
  1767  	bucketResult3 := &BucketResult{
  1768  		ElemCount:   222,
  1769  		GroupByKeys: []string{"http_method", "http_status", "weekday", "state", "gender", "app_name"},
  1770  		BucketKey:   []string{"PUT", "501", "Monday", "NH", "Femali", "sig_test"},
  1771  	}
  1772  
  1773  	// Test Sorting func. If use the limit option, only the last limit lexicographical of the <field-list> is returned in the search results
  1774  	results := append(make([]*BucketResult, 0), bucketResult1, bucketResult2, bucketResult3)
  1775  	err = statisticExpr.SortBucketResult(&results)
  1776  	assert.Nil(t, err)
  1777  	assert.Equal(t, 2, len(results))
  1778  	assert.Equal(t, bucketResult2, results[0])
  1779  	assert.Equal(t, bucketResult1, results[1])
  1780  }