github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/structs/evaluationstructs_test.go (about) 1 /* 2 Copyright 2023. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package structs 18 19 import ( 20 "testing" 21 "time" 22 23 segutils "github.com/siglens/siglens/pkg/segment/utils" 24 "github.com/siglens/siglens/pkg/utils" 25 "github.com/stretchr/testify/assert" 26 ) 27 28 func Test_ConcatExpr(t *testing.T) { 29 expr := &ConcatExpr{ 30 Atoms: []*ConcatAtom{ 31 {IsField: true, Value: "FieldWithStrings"}, 32 {IsField: false, Value: " and "}, 33 {IsField: true, Value: "FieldWithNumbers"}, 34 }, 35 } 36 37 // Test GetFields() 38 assert.Equal(t, expr.GetFields(), []string{"FieldWithStrings", "FieldWithNumbers"}) 39 40 // Test Evaluate() 41 fieldToValue := make(map[string]segutils.CValueEnclosure) 42 fieldToValue["FieldWithStrings"] = segutils.CValueEnclosure{ 43 Dtype: segutils.SS_DT_STRING, 44 CVal: "testing", 45 } 46 fieldToValue["FieldWithNumbers"] = segutils.CValueEnclosure{ 47 Dtype: segutils.SS_DT_SIGNED_NUM, 48 CVal: int64(123), 49 } 50 51 value, err := expr.Evaluate(fieldToValue) 52 assert.Nil(t, err) 53 assert.Equal(t, value, "testing and 123") 54 55 // When fieldToValue is missing fields, Evaluate() should error. 56 delete(fieldToValue, "FieldWithNumbers") 57 _, err = expr.Evaluate(fieldToValue) 58 assert.NotNil(t, err) 59 } 60 61 func Test_NumericExpr(t *testing.T) { 62 exprA := &NumericExpr{ 63 IsTerminal: true, 64 ValueIsField: false, 65 Value: "5", 66 } 67 exprB := &NumericExpr{ 68 IsTerminal: true, 69 ValueIsField: true, 70 Value: "Max", 71 NumericExprMode: NEMNumberField, 72 } 73 exprC := &NumericExpr{ 74 IsTerminal: false, 75 Op: "-", 76 Left: exprB, 77 Right: exprA, 78 } 79 exprD := &NumericExpr{ 80 IsTerminal: true, 81 ValueIsField: true, 82 Value: "Min", 83 NumericExprMode: NEMNumberField, 84 } 85 exprE := &NumericExpr{ 86 IsTerminal: false, 87 Op: "/", 88 Left: exprC, 89 Right: exprD, 90 } 91 92 // Test GetFields() 93 assert.Equal(t, exprA.GetFields(), []string{}) 94 95 assert.Equal(t, exprB.GetFields(), []string{"Max"}) 96 assert.Equal(t, exprC.GetFields(), []string{"Max"}) 97 assert.Equal(t, exprD.GetFields(), []string{"Min"}) 98 99 eFields := exprE.GetFields() 100 assert.True(t, utils.SliceContainsString(eFields, "Min")) 101 102 assert.True(t, utils.SliceContainsString(eFields, "Max")) 103 104 // Test Evaluate() 105 fieldToValue := make(map[string]segutils.CValueEnclosure) 106 fieldToValue["Min"] = segutils.CValueEnclosure{ 107 Dtype: segutils.SS_DT_FLOAT, 108 CVal: float64(12), 109 } 110 111 fieldToValue["Max"] = segutils.CValueEnclosure{ 112 Dtype: segutils.SS_DT_SIGNED_NUM, 113 CVal: int64(62), 114 } 115 116 value, err := exprA.Evaluate(fieldToValue) 117 assert.Nil(t, err) 118 assert.Equal(t, value, float64(5)) 119 120 value, err = exprB.Evaluate(fieldToValue) 121 122 assert.Nil(t, err) 123 assert.Equal(t, value, float64(62)) 124 125 value, err = exprC.Evaluate(fieldToValue) 126 assert.Nil(t, err) 127 assert.Equal(t, value, float64(62-5)) 128 129 value, err = exprD.Evaluate(fieldToValue) 130 assert.Nil(t, err) 131 assert.Equal(t, value, float64(12)) 132 133 value, err = exprE.Evaluate(fieldToValue) 134 assert.Nil(t, err) 135 assert.Equal(t, value, float64(62-5)/12) 136 137 // When fieldToValue is missing fields, Evaluate() should error. 138 delete(fieldToValue, "Max") 139 _, err = exprE.Evaluate(fieldToValue) 140 assert.NotNil(t, err) 141 142 multiplierExpr := &NumericExpr{ 143 IsTerminal: true, 144 ValueIsField: false, 145 Value: "3.14", 146 } 147 httpStatusExpr := &NumericExpr{ 148 IsTerminal: true, 149 ValueIsField: true, 150 Value: "http_status", 151 NumericExprMode: NEMNumberField, 152 } 153 productExpr := &NumericExpr{ 154 IsTerminal: false, 155 Op: "*", 156 Left: multiplierExpr, 157 Right: httpStatusExpr, 158 } 159 exactExpr := &NumericExpr{ 160 IsTerminal: false, 161 Op: "exact", 162 Left: productExpr, 163 } 164 assert.Equal(t, exactExpr.GetFields(), []string{"http_status"}) 165 166 fieldToValue["http_status"] = segutils.CValueEnclosure{ 167 Dtype: segutils.SS_DT_SIGNED_NUM, 168 CVal: int64(200), 169 } 170 171 value, err = exactExpr.Evaluate(fieldToValue) 172 assert.Nil(t, err) 173 assert.Equal(t, value, float64(628)) 174 175 expExpr := &NumericExpr{ 176 IsTerminal: false, 177 Op: "exp", 178 Left: &NumericExpr{ 179 IsTerminal: true, 180 ValueIsField: false, 181 Value: "3", 182 }, 183 } 184 185 assert.Equal(t, expExpr.GetFields(), []string{}) 186 187 value, err = expExpr.Evaluate(fieldToValue) 188 assert.Nil(t, err) 189 assert.Equal(t, value, 20.085536923187668) 190 191 nowExpr := &NumericExpr{ 192 NumericExprMode: NEMNumber, 193 IsTerminal: true, 194 Op: "now", 195 } 196 assert.Equal(t, nowExpr.GetFields(), []string{}) 197 198 // Test Evaluate() 199 fieldToValue = make(map[string]segutils.CValueEnclosure) 200 201 value, err = nowExpr.Evaluate(fieldToValue) 202 assert.Nil(t, err) 203 currentTimestamp := time.Now().Unix() 204 205 assert.InDelta(t, currentTimestamp, value, 1, "The evaluated timestamp is not within the expected range") 206 207 strToNumber := 208 &NumericExpr{ 209 NumericExprMode: NEMNumericExpr, 210 IsTerminal: false, 211 Op: "tonumber", 212 Right: &NumericExpr{ 213 NumericExprMode: NEMNumber, 214 IsTerminal: true, 215 ValueIsField: false, 216 Value: "16", 217 }, 218 Val: &StringExpr{ 219 StringExprMode: SEMRawString, 220 RawString: "0A4", 221 }, 222 } 223 assert.Equal(t, strToNumber.GetFields(), []string{}) 224 225 value, err = strToNumber.Evaluate(fieldToValue) 226 assert.Nil(t, err) 227 assert.Equal(t, value, float64(164)) 228 229 } 230 231 func Test_ValueExpr(t *testing.T) { 232 numericExpr := &NumericExpr{ 233 IsTerminal: false, 234 Op: "-", 235 Left: &NumericExpr{ 236 IsTerminal: true, 237 ValueIsField: true, 238 Value: "Max", 239 NumericExprMode: NEMNumberField, 240 }, 241 Right: &NumericExpr{ 242 IsTerminal: true, 243 ValueIsField: false, 244 Value: "5", 245 }, 246 } 247 248 concatExpr := &ConcatExpr{ 249 Atoms: []*ConcatAtom{ 250 {IsField: true, Value: "FieldWithStrings"}, 251 {IsField: false, Value: " and "}, 252 {IsField: true, Value: "FieldWithNumbers"}, 253 }, 254 } 255 256 valueExprA := &ValueExpr{ 257 ValueExprMode: VEMNumericExpr, 258 NumericExpr: numericExpr, 259 } 260 261 valueExprB := &ValueExpr{ 262 ValueExprMode: VEMStringExpr, 263 StringExpr: &StringExpr{ 264 StringExprMode: SEMConcatExpr, 265 ConcatExpr: concatExpr, 266 }, 267 } 268 269 valueExprC := &ValueExpr{ 270 ValueExprMode: VEMStringExpr, 271 StringExpr: &StringExpr{ 272 StringExprMode: SEMConcatExpr, 273 ConcatExpr: &ConcatExpr{ 274 Atoms: []*ConcatAtom{ 275 {Value: "hello"}, 276 }, 277 }, 278 }, 279 } 280 valueExprD := &ValueExpr{ 281 ValueExprMode: VEMNumericExpr, 282 NumericExpr: &NumericExpr{ 283 284 IsTerminal: true, 285 ValueIsField: false, 286 Value: "99", 287 }, 288 } 289 290 valueExprE := &ValueExpr{ 291 ValueExprMode: VEMNumericExpr, 292 NumericExpr: &NumericExpr{ 293 IsTerminal: true, 294 ValueIsField: true, 295 296 Value: "Seconds", 297 NumericExprMode: NEMNumberField, 298 }, 299 } 300 301 // Test GetFields() 302 assert.Equal(t, valueExprA.GetFields(), []string{"Max"}) 303 assert.Equal(t, valueExprB.GetFields(), []string{"FieldWithStrings", "FieldWithNumbers"}) 304 assert.Equal(t, valueExprC.GetFields(), []string{}) 305 assert.Equal(t, valueExprD.GetFields(), []string{}) 306 assert.Equal(t, valueExprE.GetFields(), []string{"Seconds"}) 307 308 // Test Evaluate() 309 fieldToValue := make(map[string]segutils.CValueEnclosure) 310 fieldToValue["Max"] = segutils.CValueEnclosure{ 311 Dtype: segutils.SS_DT_FLOAT, 312 CVal: float64(62), 313 } 314 fieldToValue["Seconds"] = segutils.CValueEnclosure{ 315 Dtype: segutils.SS_DT_SIGNED_NUM, 316 CVal: int64(42), 317 } 318 fieldToValue["FieldWithStrings"] = segutils.CValueEnclosure{ 319 Dtype: segutils.SS_DT_STRING, 320 CVal: "testing", 321 } 322 fieldToValue["FieldWithNumbers"] = segutils.CValueEnclosure{ 323 Dtype: segutils.SS_DT_SIGNED_NUM, 324 CVal: int64(123), 325 } 326 327 var valueFloat float64 328 var valueStr string 329 var err error 330 valueFloat, err = valueExprA.EvaluateToFloat(fieldToValue) 331 assert.Nil(t, err) 332 assert.Equal(t, valueFloat, float64(62-5)) 333 valueStr, err = valueExprA.EvaluateToString(fieldToValue) 334 assert.Nil(t, err) 335 assert.Equal(t, valueStr, "57") 336 337 _, err = valueExprB.EvaluateToFloat(fieldToValue) 338 assert.NotNil(t, err) 339 valueStr, err = valueExprB.EvaluateToString(fieldToValue) 340 assert.Nil(t, err) 341 assert.Equal(t, valueStr, "testing and 123") 342 343 _, err = valueExprC.EvaluateToFloat(fieldToValue) 344 assert.NotNil(t, err) 345 valueStr, err = valueExprC.EvaluateToString(fieldToValue) 346 assert.Nil(t, err) 347 assert.Equal(t, valueStr, "hello") 348 349 valueFloat, err = valueExprD.EvaluateToFloat(fieldToValue) 350 assert.Nil(t, err) 351 assert.Equal(t, valueFloat, float64(99)) 352 valueStr, err = valueExprD.EvaluateToString(fieldToValue) 353 assert.Nil(t, err) 354 assert.Equal(t, valueStr, "99") 355 356 valueFloat, err = valueExprE.EvaluateToFloat(fieldToValue) 357 assert.Nil(t, err) 358 assert.Equal(t, valueFloat, float64(42)) 359 valueStr, err = valueExprE.EvaluateToString(fieldToValue) 360 assert.Nil(t, err) 361 assert.Equal(t, valueStr, "42") 362 363 // When fieldToValue is missing fields, Evaluate() should error. 364 fieldToValue = make(map[string]segutils.CValueEnclosure) 365 _, err = valueExprA.EvaluateToFloat(fieldToValue) 366 assert.NotNil(t, err) 367 _, err = valueExprA.EvaluateToString(fieldToValue) 368 assert.NotNil(t, err) 369 370 _, err = valueExprB.EvaluateToFloat(fieldToValue) 371 assert.NotNil(t, err) 372 _, err = valueExprA.EvaluateToString(fieldToValue) 373 assert.NotNil(t, err) 374 375 _, err = valueExprE.EvaluateToFloat(fieldToValue) 376 assert.NotNil(t, err) 377 _, err = valueExprA.EvaluateToString(fieldToValue) 378 assert.NotNil(t, err) 379 } 380 381 func Test_BoolExpr(t *testing.T) { 382 valueExprA := &ValueExpr{ 383 ValueExprMode: VEMNumericExpr, 384 NumericExpr: &NumericExpr{ 385 IsTerminal: false, 386 Op: "-", 387 Left: &NumericExpr{ 388 IsTerminal: true, 389 ValueIsField: true, 390 Value: "Max", 391 NumericExprMode: NEMNumberField, 392 }, 393 Right: &NumericExpr{ 394 IsTerminal: true, 395 ValueIsField: false, 396 Value: "5", 397 }, 398 }, 399 } 400 401 valueExprB := &ValueExpr{ 402 ValueExprMode: VEMStringExpr, 403 StringExpr: &StringExpr{ 404 StringExprMode: SEMConcatExpr, 405 ConcatExpr: &ConcatExpr{ 406 Atoms: []*ConcatAtom{ 407 {IsField: true, Value: "FieldWithStrings"}, 408 {IsField: false, Value: " and "}, 409 {IsField: true, Value: "FieldWithNumbers"}, 410 }, 411 }, 412 }, 413 } 414 415 valueExprC := &ValueExpr{ 416 ValueExprMode: VEMStringExpr, 417 StringExpr: &StringExpr{ 418 StringExprMode: SEMConcatExpr, 419 ConcatExpr: &ConcatExpr{ 420 421 Atoms: []*ConcatAtom{ 422 {Value: "hello"}, 423 }, 424 }, 425 }, 426 } 427 428 valueExprD := &ValueExpr{ 429 ValueExprMode: VEMNumericExpr, 430 NumericExpr: &NumericExpr{ 431 432 IsTerminal: true, 433 ValueIsField: false, 434 Value: "99", 435 }, 436 } 437 438 boolExprA := &BoolExpr{ 439 IsTerminal: true, 440 ValueOp: "<", 441 LeftValue: valueExprA, 442 RightValue: valueExprD, 443 } 444 445 boolExprB := &BoolExpr{ 446 IsTerminal: true, 447 ValueOp: "=", 448 LeftValue: valueExprB, 449 RightValue: valueExprC, 450 } 451 452 boolExprC := &BoolExpr{ 453 IsTerminal: false, 454 BoolOp: BoolOpOr, 455 LeftBool: boolExprA, 456 RightBool: boolExprB, 457 } 458 459 boolExprD := &BoolExpr{ 460 IsTerminal: false, 461 BoolOp: BoolOpAnd, 462 LeftBool: boolExprA, 463 RightBool: boolExprB, 464 } 465 466 boolExprE := &BoolExpr{ 467 IsTerminal: false, 468 BoolOp: BoolOpNot, 469 LeftBool: boolExprA, 470 } 471 472 // Test GetFields() 473 assert.Equal(t, boolExprA.GetFields(), []string{"Max"}) 474 assert.Equal(t, boolExprB.GetFields(), []string{"FieldWithStrings", "FieldWithNumbers"}) 475 476 cFields := boolExprC.GetFields() 477 assert.True(t, utils.SliceContainsString(cFields, "Max")) 478 assert.True(t, utils.SliceContainsString(cFields, "FieldWithStrings")) 479 assert.True(t, utils.SliceContainsString(cFields, "FieldWithNumbers")) 480 481 dFields := boolExprD.GetFields() 482 assert.True(t, utils.SliceContainsString(dFields, "Max")) 483 assert.True(t, utils.SliceContainsString(dFields, "FieldWithStrings")) 484 assert.True(t, utils.SliceContainsString(dFields, "FieldWithNumbers")) 485 486 assert.Equal(t, boolExprE.GetFields(), []string{"Max"}) 487 488 // Test Evaluate() 489 fieldToValue := make(map[string]segutils.CValueEnclosure) 490 fieldToValue["Max"] = segutils.CValueEnclosure{ 491 Dtype: segutils.SS_DT_FLOAT, 492 CVal: float64(62), 493 } 494 fieldToValue["FieldWithStrings"] = segutils.CValueEnclosure{ 495 Dtype: segutils.SS_DT_STRING, 496 CVal: "testing", 497 } 498 fieldToValue["FieldWithNumbers"] = segutils.CValueEnclosure{ 499 Dtype: segutils.SS_DT_SIGNED_NUM, 500 CVal: int64(123), 501 } 502 503 value, err := boolExprA.Evaluate(fieldToValue) 504 505 assert.Nil(t, err) 506 assert.Equal(t, value, true) 507 508 value, err = boolExprB.Evaluate(fieldToValue) 509 assert.Nil(t, err) 510 assert.Equal(t, value, false) 511 512 value, err = boolExprC.Evaluate(fieldToValue) 513 assert.Nil(t, err) 514 assert.Equal(t, value, true) 515 516 value, err = boolExprD.Evaluate(fieldToValue) 517 assert.Nil(t, err) 518 assert.Equal(t, value, false) 519 520 value, err = boolExprE.Evaluate(fieldToValue) 521 assert.Nil(t, err) 522 assert.Equal(t, value, false) 523 524 // In a terminal node, the left and right ValueExpr must both be strings or both be floats. 525 boolExprBadValueExprTypes := &BoolExpr{ 526 IsTerminal: true, 527 ValueOp: "!=", 528 LeftValue: valueExprA, // Evaluates to float 529 RightValue: valueExprB, // Evaluates to string 530 } 531 532 value, err = boolExprBadValueExprTypes.Evaluate(fieldToValue) 533 assert.Nil(t, err) 534 assert.Equal(t, value, true) 535 536 // In a terminal node with string ValueExprs, the ValueOp cannot be an inequality. 537 boolExprBadOpForStringValues := &BoolExpr{ 538 IsTerminal: true, 539 ValueOp: "<", 540 LeftValue: valueExprB, // Evaluates to string 541 RightValue: valueExprC, // Evaluates to string 542 } 543 544 _, err = boolExprBadOpForStringValues.Evaluate(fieldToValue) 545 assert.NotNil(t, err) 546 547 // When fieldToValue is missing fields, Evaluate() should error. 548 delete(fieldToValue, "Max") 549 delete(fieldToValue, "FieldWithNumbers") 550 _, err = boolExprA.Evaluate(fieldToValue) 551 assert.NotNil(t, err) 552 553 _, err = boolExprB.Evaluate(fieldToValue) 554 assert.NotNil(t, err) 555 556 _, err = boolExprC.Evaluate(fieldToValue) 557 assert.NotNil(t, err) 558 559 _, err = boolExprD.Evaluate(fieldToValue) 560 assert.NotNil(t, err) 561 562 _, err = boolExprE.Evaluate(fieldToValue) 563 assert.NotNil(t, err) 564 } 565 566 func Test_ConditionExpr(t *testing.T) { 567 568 boolExpr := 569 &BoolExpr{ 570 IsTerminal: true, 571 LeftValue: &ValueExpr{ 572 ValueExprMode: VEMNumericExpr, 573 NumericExpr: &NumericExpr{ 574 NumericExprMode: NEMNumberField, 575 IsTerminal: true, 576 ValueIsField: true, 577 Value: "state", 578 }, 579 }, 580 ValueOp: "in", 581 ValueList: []*ValueExpr{ 582 &ValueExpr{ 583 ValueExprMode: VEMStringExpr, 584 StringExpr: &StringExpr{ 585 StringExprMode: SEMConcatExpr, 586 ConcatExpr: &ConcatExpr{ 587 Atoms: []*ConcatAtom{ 588 {IsField: false, Value: "Mary"}, 589 {IsField: false, Value: "land"}, 590 }, 591 }, 592 }, 593 }, 594 &ValueExpr{ 595 ValueExprMode: VEMStringExpr, 596 StringExpr: &StringExpr{ 597 StringExprMode: SEMRawString, 598 RawString: "Hawaii", 599 }, 600 }, 601 &ValueExpr{ 602 ValueExprMode: VEMNumericExpr, 603 NumericExpr: &NumericExpr{ 604 NumericExprMode: NEMNumericExpr, 605 Op: "+", 606 Left: &NumericExpr{ 607 NumericExprMode: NEMNumber, 608 IsTerminal: true, 609 ValueIsField: false, 610 Value: "99", 611 }, 612 Right: &NumericExpr{ 613 NumericExprMode: NEMNumber, 614 IsTerminal: true, 615 ValueIsField: false, 616 Value: "1", 617 }, 618 }, 619 }, 620 }, 621 } 622 623 conditionExpr := &ConditionExpr{ 624 Op: "if", 625 BoolExpr: boolExpr, 626 TrueValue: &ValueExpr{ 627 ValueExprMode: VEMNumericExpr, 628 NumericExpr: &NumericExpr{ 629 NumericExprMode: NEMNumberField, 630 IsTerminal: true, 631 ValueIsField: true, 632 Value: "state", 633 }, 634 }, 635 FalseValue: &ValueExpr{ 636 ValueExprMode: VEMStringExpr, 637 StringExpr: &StringExpr{ 638 StringExprMode: SEMRawString, 639 RawString: "Error", 640 }, 641 }, 642 } 643 644 // Test GetFields() 645 assert.Equal(t, boolExpr.GetFields(), []string{"state"}) 646 647 // Test Evaluate() 648 fieldToValue := make(map[string]segutils.CValueEnclosure) 649 fieldToValue["state"] = segutils.CValueEnclosure{ 650 Dtype: segutils.SS_DT_STRING, 651 CVal: "Maryland", 652 } 653 654 value, err := boolExpr.Evaluate(fieldToValue) 655 assert.Nil(t, err) 656 assert.Equal(t, value, true) 657 658 fieldToValue["state"] = segutils.CValueEnclosure{ 659 Dtype: segutils.SS_DT_STRING, 660 CVal: "MarylandTest", 661 } 662 663 value, err = boolExpr.Evaluate(fieldToValue) 664 assert.Nil(t, err) 665 assert.Equal(t, value, false) 666 667 fieldToValue["state"] = segutils.CValueEnclosure{ 668 Dtype: segutils.SS_DT_STRING, 669 CVal: "Hawaii", 670 } 671 672 //If true, it should return true value: state 673 str, err := conditionExpr.EvaluateCondition(fieldToValue) 674 assert.Nil(t, err) 675 assert.Equal(t, str, "Hawaii") 676 677 fieldToValue["state"] = segutils.CValueEnclosure{ 678 Dtype: segutils.SS_DT_STRING, 679 CVal: "NewYork", 680 } 681 682 //If false, it should return false value: "Error" 683 str, err = conditionExpr.EvaluateCondition(fieldToValue) 684 assert.Nil(t, err) 685 assert.Equal(t, str, "Error") 686 687 fieldToValue["state"] = segutils.CValueEnclosure{ 688 Dtype: segutils.SS_DT_STRING, 689 CVal: "Maryland", 690 } 691 692 str, err = conditionExpr.EvaluateCondition(fieldToValue) 693 assert.Nil(t, err) 694 assert.Equal(t, str, "Maryland") 695 696 boolExpr.LeftValue.NumericExpr.Value = "101" 697 boolExpr.LeftValue.NumericExpr.ValueIsField = false 698 boolExpr.LeftValue.NumericExpr.NumericExprMode = NEMNumber 699 value, err = boolExpr.Evaluate(fieldToValue) 700 assert.Nil(t, err) 701 assert.Equal(t, value, false) 702 703 boolExpr.LeftValue.NumericExpr.Value = "100" 704 value, err = boolExpr.Evaluate(fieldToValue) 705 assert.Nil(t, err) 706 assert.Equal(t, value, true) 707 708 isStr := 709 &BoolExpr{ 710 IsTerminal: true, 711 LeftValue: &ValueExpr{ 712 ValueExprMode: VEMNumericExpr, 713 NumericExpr: &NumericExpr{ 714 NumericExprMode: NEMNumberField, 715 IsTerminal: true, 716 ValueIsField: true, 717 Value: "country", 718 }, 719 }, 720 ValueOp: "isstr", 721 } 722 isStrIf := 723 &ConditionExpr{ 724 Op: "if", 725 BoolExpr: boolExpr, 726 TrueValue: &ValueExpr{ 727 ValueExprMode: VEMStringExpr, 728 StringExpr: &StringExpr{ 729 StringExprMode: SEMRawString, 730 RawString: "This is a string", 731 }, 732 }, 733 FalseValue: &ValueExpr{ 734 ValueExprMode: VEMStringExpr, 735 StringExpr: &StringExpr{ 736 StringExprMode: SEMRawString, 737 RawString: "This is not a string", 738 }, 739 }, 740 } 741 742 assert.Equal(t, isStr.GetFields(), []string{"country"}) 743 fieldToValue["country"] = segutils.CValueEnclosure{ 744 Dtype: segutils.SS_DT_STRING, 745 CVal: "Spain", 746 } 747 748 str, err = isStrIf.EvaluateCondition(fieldToValue) 749 assert.Nil(t, err) 750 assert.Equal(t, str, "This is a string") 751 752 isNotStr := 753 &BoolExpr{ 754 IsTerminal: true, 755 LeftValue: &ValueExpr{ 756 ValueExprMode: VEMNumericExpr, 757 NumericExpr: &NumericExpr{ 758 NumericExprMode: NEMNumberField, 759 IsTerminal: true, 760 ValueIsField: true, 761 Value: "longitude", 762 }, 763 }, 764 ValueOp: "isstr", 765 } 766 isNotStrIf := 767 &ConditionExpr{ 768 Op: "if", 769 BoolExpr: isNotStr, 770 TrueValue: &ValueExpr{ 771 ValueExprMode: VEMStringExpr, 772 StringExpr: &StringExpr{ 773 StringExprMode: SEMRawString, 774 RawString: "This is a string", 775 }, 776 }, 777 FalseValue: &ValueExpr{ 778 ValueExprMode: VEMStringExpr, 779 StringExpr: &StringExpr{ 780 StringExprMode: SEMRawString, 781 RawString: "This is not a string", 782 }, 783 }, 784 } 785 786 assert.Equal(t, isNotStr.GetFields(), []string{"longitude"}) 787 fieldToValue["longitude"] = segutils.CValueEnclosure{ 788 Dtype: segutils.SS_DT_FLOAT, 789 CVal: float64(99.619024), 790 } 791 792 str, err = isNotStrIf.EvaluateCondition(fieldToValue) 793 assert.Nil(t, err) 794 assert.Equal(t, str, "This is not a string") 795 796 isInt := 797 &BoolExpr{ 798 IsTerminal: true, 799 LeftValue: &ValueExpr{ 800 ValueExprMode: VEMNumericExpr, 801 NumericExpr: &NumericExpr{ 802 NumericExprMode: NEMNumberField, 803 IsTerminal: true, 804 ValueIsField: true, 805 Value: "http_status", 806 }, 807 }, 808 ValueOp: "isint", 809 } 810 isIntIf := 811 &ConditionExpr{ 812 Op: "if", 813 BoolExpr: isInt, 814 TrueValue: &ValueExpr{ 815 ValueExprMode: VEMStringExpr, 816 StringExpr: &StringExpr{ 817 StringExprMode: SEMRawString, 818 RawString: "This is an integer", 819 }, 820 }, 821 FalseValue: &ValueExpr{ 822 ValueExprMode: VEMStringExpr, 823 StringExpr: &StringExpr{ 824 StringExprMode: SEMRawString, 825 RawString: "This is not an integer", 826 }, 827 }, 828 } 829 830 assert.Equal(t, isInt.GetFields(), []string{"http_status"}) 831 fieldToValue["http_status"] = segutils.CValueEnclosure{ 832 Dtype: segutils.SS_DT_SIGNED_NUM, 833 CVal: int64(500), 834 } 835 836 str, err = isIntIf.EvaluateCondition(fieldToValue) 837 assert.Nil(t, err) 838 assert.Equal(t, str, "This is an integer") 839 840 isNotInt := 841 &BoolExpr{ 842 IsTerminal: true, 843 LeftValue: &ValueExpr{ 844 ValueExprMode: VEMNumericExpr, 845 NumericExpr: &NumericExpr{ 846 NumericExprMode: NEMNumberField, 847 IsTerminal: true, 848 ValueIsField: true, 849 Value: "longitude", 850 }, 851 }, 852 ValueOp: "isint", 853 } 854 isNotIntIf := 855 &ConditionExpr{ 856 Op: "if", 857 BoolExpr: isNotInt, 858 TrueValue: &ValueExpr{ 859 ValueExprMode: VEMStringExpr, 860 StringExpr: &StringExpr{ 861 StringExprMode: SEMRawString, 862 RawString: "This is an integer", 863 }, 864 }, 865 FalseValue: &ValueExpr{ 866 ValueExprMode: VEMStringExpr, 867 StringExpr: &StringExpr{ 868 StringExprMode: SEMRawString, 869 RawString: "This is not an integer", 870 }, 871 }, 872 } 873 874 assert.Equal(t, isNotInt.GetFields(), []string{"longitude"}) 875 fieldToValue["longitude"] = segutils.CValueEnclosure{ 876 Dtype: segutils.SS_DT_FLOAT, 877 CVal: float64(99.619024), 878 } 879 880 str, err = isNotIntIf.EvaluateCondition(fieldToValue) 881 assert.Nil(t, err) 882 assert.Equal(t, str, "This is not an integer") 883 884 isNotBool := 885 &BoolExpr{ 886 IsTerminal: true, 887 LeftValue: &ValueExpr{ 888 ValueExprMode: VEMNumericExpr, 889 NumericExpr: &NumericExpr{ 890 NumericExprMode: NEMNumberField, 891 IsTerminal: true, 892 ValueIsField: true, 893 Value: "city", 894 }, 895 }, 896 ValueOp: "isbool", 897 } 898 isNotBoolIf := 899 &ConditionExpr{ 900 Op: "if", 901 BoolExpr: isNotBool, 902 TrueValue: &ValueExpr{ 903 ValueExprMode: VEMStringExpr, 904 StringExpr: &StringExpr{ 905 StringExprMode: SEMRawString, 906 RawString: "This is a boolean value", 907 }, 908 }, 909 FalseValue: &ValueExpr{ 910 ValueExprMode: VEMStringExpr, 911 StringExpr: &StringExpr{ 912 StringExprMode: SEMRawString, 913 RawString: "This is not a boolean value", 914 }, 915 }, 916 } 917 918 assert.Equal(t, isNotBool.GetFields(), []string{"city"}) 919 fieldToValue["city"] = segutils.CValueEnclosure{ 920 Dtype: segutils.SS_DT_STRING, 921 CVal: "Boston", 922 } 923 924 str, err = isNotBoolIf.EvaluateCondition(fieldToValue) 925 assert.Nil(t, err) 926 assert.Equal(t, str, "This is not a boolean value") 927 928 isBool := 929 &BoolExpr{ 930 IsTerminal: true, 931 LeftValue: &ValueExpr{ 932 ValueExprMode: VEMNumericExpr, 933 NumericExpr: &NumericExpr{ 934 NumericExprMode: NEMNumber, 935 IsTerminal: true, 936 ValueIsField: false, 937 Value: "true", 938 }, 939 }, 940 ValueOp: "isbool", 941 } 942 isBoolIf := 943 &ConditionExpr{ 944 Op: "if", 945 BoolExpr: isBool, 946 TrueValue: &ValueExpr{ 947 ValueExprMode: VEMStringExpr, 948 StringExpr: &StringExpr{ 949 StringExprMode: SEMRawString, 950 RawString: "This is a boolean value", 951 }, 952 }, 953 FalseValue: &ValueExpr{ 954 ValueExprMode: VEMStringExpr, 955 StringExpr: &StringExpr{ 956 StringExprMode: SEMRawString, 957 RawString: "This is not a boolean value", 958 }, 959 }, 960 } 961 962 assert.Equal(t, isBool.GetFields(), []string{}) 963 fieldToValue["true"] = segutils.CValueEnclosure{ 964 Dtype: segutils.SS_DT_STRING, 965 CVal: "true", 966 } 967 968 str, err = isBoolIf.EvaluateCondition(fieldToValue) 969 assert.Nil(t, err) 970 assert.Equal(t, str, "This is a boolean value") 971 972 IsNotNull := 973 &BoolExpr{ 974 IsTerminal: true, 975 LeftValue: &ValueExpr{ 976 ValueExprMode: VEMNumericExpr, 977 NumericExpr: &NumericExpr{ 978 NumericExprMode: NEMNumberField, 979 IsTerminal: true, 980 ValueIsField: true, 981 Value: "state", 982 }, 983 }, 984 ValueOp: "isbool", 985 } 986 isNotNullIf := 987 &ConditionExpr{ 988 Op: "if", 989 BoolExpr: IsNotNull, 990 TrueValue: &ValueExpr{ 991 ValueExprMode: VEMStringExpr, 992 StringExpr: &StringExpr{ 993 StringExprMode: SEMRawString, 994 RawString: "This is a null value", 995 }, 996 }, 997 FalseValue: &ValueExpr{ 998 ValueExprMode: VEMStringExpr, 999 StringExpr: &StringExpr{ 1000 StringExprMode: SEMRawString, 1001 RawString: "This is not a null value", 1002 }, 1003 }, 1004 } 1005 1006 assert.Equal(t, IsNotNull.GetFields(), []string{"state"}) 1007 fieldToValue["state"] = segutils.CValueEnclosure{ 1008 Dtype: segutils.SS_DT_STRING, 1009 CVal: "Florida", 1010 } 1011 1012 str, err = isNotNullIf.EvaluateCondition(fieldToValue) 1013 assert.Nil(t, err) 1014 assert.Equal(t, str, "This is not a null value") 1015 1016 cidrBoolExpr := 1017 &BoolExpr{ 1018 IsTerminal: true, 1019 LeftValue: &ValueExpr{ 1020 ValueExprMode: VEMStringExpr, 1021 StringExpr: &StringExpr{ 1022 StringExprMode: SEMRawString, 1023 RawString: "192.0.2.0/24", 1024 }, 1025 }, 1026 RightValue: &ValueExpr{ 1027 ValueExprMode: VEMStringExpr, 1028 StringExpr: &StringExpr{ 1029 StringExprMode: SEMRawString, 1030 RawString: "192.0.2.5", 1031 }, 1032 }, 1033 ValueOp: "cidrmatch", 1034 } 1035 cidrConditionExpr := 1036 &ConditionExpr{ 1037 Op: "if", 1038 BoolExpr: cidrBoolExpr, 1039 TrueValue: &ValueExpr{ 1040 ValueExprMode: VEMStringExpr, 1041 StringExpr: &StringExpr{ 1042 StringExprMode: SEMRawString, 1043 RawString: "local", 1044 }, 1045 }, 1046 FalseValue: &ValueExpr{ 1047 ValueExprMode: VEMStringExpr, 1048 StringExpr: &StringExpr{ 1049 StringExprMode: SEMRawString, 1050 RawString: "not local", 1051 }, 1052 }, 1053 } 1054 1055 assert.Equal(t, cidrBoolExpr.GetFields(), []string{}) 1056 1057 str, err = cidrConditionExpr.EvaluateCondition(fieldToValue) 1058 assert.Nil(t, err) 1059 assert.Equal(t, str, "local") 1060 1061 notCidrMatchBoolExpr := 1062 &BoolExpr{ 1063 IsTerminal: true, 1064 LeftValue: &ValueExpr{ 1065 ValueExprMode: VEMStringExpr, 1066 StringExpr: &StringExpr{ 1067 StringExprMode: SEMRawString, 1068 RawString: "192.0.2.0/24", 1069 }, 1070 }, 1071 RightValue: &ValueExpr{ 1072 ValueExprMode: VEMStringExpr, 1073 StringExpr: &StringExpr{ 1074 StringExprMode: SEMRawString, 1075 RawString: "192.0.3.1", 1076 }, 1077 }, 1078 ValueOp: "cidrmatch", 1079 } 1080 notCidrMatchConditionExpr := 1081 &ConditionExpr{ 1082 Op: "if", 1083 BoolExpr: notCidrMatchBoolExpr, 1084 TrueValue: &ValueExpr{ 1085 ValueExprMode: VEMStringExpr, 1086 StringExpr: &StringExpr{ 1087 StringExprMode: SEMRawString, 1088 RawString: "local", 1089 }, 1090 }, 1091 FalseValue: &ValueExpr{ 1092 ValueExprMode: VEMStringExpr, 1093 StringExpr: &StringExpr{ 1094 StringExprMode: SEMRawString, 1095 RawString: "not local", 1096 }, 1097 }, 1098 } 1099 assert.Equal(t, notCidrMatchBoolExpr.GetFields(), []string{}) 1100 1101 str, err = notCidrMatchConditionExpr.EvaluateCondition(fieldToValue) 1102 assert.Nil(t, err) 1103 assert.Equal(t, str, "not local") 1104 1105 likeBoolExpr := 1106 &BoolExpr{ 1107 IsTerminal: true, 1108 LeftValue: &ValueExpr{ 1109 ValueExprMode: VEMNumericExpr, 1110 NumericExpr: &NumericExpr{ 1111 NumericExprMode: NEMNumberField, 1112 IsTerminal: true, 1113 ValueIsField: true, 1114 Value: "http_status", 1115 }, 1116 }, 1117 RightValue: &ValueExpr{ 1118 ValueExprMode: VEMStringExpr, 1119 StringExpr: &StringExpr{ 1120 StringExprMode: SEMRawString, 1121 RawString: "4%", 1122 }, 1123 }, 1124 ValueOp: "like", 1125 } 1126 likeConditionExpr := 1127 &ConditionExpr{ 1128 Op: "if", 1129 BoolExpr: likeBoolExpr, 1130 TrueValue: &ValueExpr{ 1131 ValueExprMode: VEMStringExpr, 1132 StringExpr: &StringExpr{ 1133 StringExprMode: SEMRawString, 1134 RawString: "True", 1135 }, 1136 }, 1137 FalseValue: &ValueExpr{ 1138 ValueExprMode: VEMStringExpr, 1139 StringExpr: &StringExpr{ 1140 StringExprMode: SEMRawString, 1141 RawString: "False", 1142 }, 1143 }, 1144 } 1145 1146 assert.Equal(t, likeBoolExpr.GetFields(), []string{"http_status"}) 1147 fieldToValue["http_status"] = segutils.CValueEnclosure{ 1148 Dtype: segutils.SS_DT_SIGNED_NUM, 1149 CVal: int64(400), 1150 } 1151 1152 str, err = likeConditionExpr.EvaluateCondition(fieldToValue) 1153 assert.Nil(t, err) 1154 assert.Equal(t, str, "True") 1155 1156 notLikeBoolExpr := 1157 &BoolExpr{ 1158 IsTerminal: true, 1159 LeftValue: &ValueExpr{ 1160 ValueExprMode: VEMNumericExpr, 1161 NumericExpr: &NumericExpr{ 1162 NumericExprMode: NEMNumberField, 1163 IsTerminal: true, 1164 ValueIsField: true, 1165 Value: "http_status", 1166 }, 1167 }, 1168 RightValue: &ValueExpr{ 1169 ValueExprMode: VEMStringExpr, 1170 StringExpr: &StringExpr{ 1171 StringExprMode: SEMRawString, 1172 RawString: "4%", 1173 }, 1174 }, 1175 ValueOp: "like", 1176 } 1177 notLikeConditionExpr := 1178 &ConditionExpr{ 1179 Op: "if", 1180 BoolExpr: likeBoolExpr, 1181 TrueValue: &ValueExpr{ 1182 ValueExprMode: VEMStringExpr, 1183 StringExpr: &StringExpr{ 1184 StringExprMode: SEMRawString, 1185 RawString: "True", 1186 }, 1187 }, 1188 FalseValue: &ValueExpr{ 1189 ValueExprMode: VEMStringExpr, 1190 StringExpr: &StringExpr{ 1191 StringExprMode: SEMRawString, 1192 RawString: "False", 1193 }, 1194 }, 1195 } 1196 1197 assert.Equal(t, notLikeBoolExpr.GetFields(), []string{"http_status"}) 1198 fieldToValue["http_status"] = segutils.CValueEnclosure{ 1199 Dtype: segutils.SS_DT_SIGNED_NUM, 1200 CVal: int64(200), 1201 } 1202 1203 str, err = notLikeConditionExpr.EvaluateCondition(fieldToValue) 1204 assert.Nil(t, err) 1205 assert.Equal(t, str, "False") 1206 1207 matchBoolExpr := 1208 &BoolExpr{ 1209 IsTerminal: true, 1210 LeftValue: &ValueExpr{ 1211 ValueExprMode: VEMNumericExpr, 1212 NumericExpr: &NumericExpr{ 1213 NumericExprMode: NEMNumberField, 1214 IsTerminal: true, 1215 ValueIsField: true, 1216 Value: "country", 1217 }, 1218 }, 1219 RightValue: &ValueExpr{ 1220 ValueExprMode: VEMStringExpr, 1221 StringExpr: &StringExpr{ 1222 StringExprMode: SEMRawString, 1223 RawString: "^Sa", 1224 }, 1225 }, 1226 ValueOp: "match", 1227 } 1228 matchConditionExpr := 1229 &ConditionExpr{ 1230 Op: "if", 1231 BoolExpr: matchBoolExpr, 1232 TrueValue: &ValueExpr{ 1233 ValueExprMode: VEMStringExpr, 1234 StringExpr: &StringExpr{ 1235 StringExprMode: SEMRawString, 1236 RawString: "yes", 1237 }, 1238 }, 1239 FalseValue: &ValueExpr{ 1240 ValueExprMode: VEMStringExpr, 1241 StringExpr: &StringExpr{ 1242 StringExprMode: SEMRawString, 1243 RawString: "no", 1244 }, 1245 }, 1246 } 1247 1248 assert.Equal(t, matchBoolExpr.GetFields(), []string{"country"}) 1249 fieldToValue["country"] = segutils.CValueEnclosure{ 1250 Dtype: segutils.SS_DT_STRING, 1251 CVal: "Saudi Arabia", 1252 } 1253 1254 str, err = matchConditionExpr.EvaluateCondition(fieldToValue) 1255 assert.Nil(t, err) 1256 assert.Equal(t, str, "yes") 1257 1258 notMatchBoolExpr := 1259 &BoolExpr{ 1260 IsTerminal: true, 1261 LeftValue: &ValueExpr{ 1262 ValueExprMode: VEMNumericExpr, 1263 NumericExpr: &NumericExpr{ 1264 NumericExprMode: NEMNumberField, 1265 IsTerminal: true, 1266 ValueIsField: true, 1267 Value: "country", 1268 }, 1269 }, 1270 RightValue: &ValueExpr{ 1271 ValueExprMode: VEMStringExpr, 1272 StringExpr: &StringExpr{ 1273 StringExprMode: SEMRawString, 1274 RawString: "^Sa", 1275 }, 1276 }, 1277 ValueOp: "match", 1278 } 1279 notMatchConditionExpr := 1280 &ConditionExpr{ 1281 Op: "if", 1282 BoolExpr: notMatchBoolExpr, 1283 TrueValue: &ValueExpr{ 1284 ValueExprMode: VEMStringExpr, 1285 StringExpr: &StringExpr{ 1286 StringExprMode: SEMRawString, 1287 RawString: "yes", 1288 }, 1289 }, 1290 FalseValue: &ValueExpr{ 1291 ValueExprMode: VEMStringExpr, 1292 StringExpr: &StringExpr{ 1293 StringExprMode: SEMRawString, 1294 RawString: "no", 1295 }, 1296 }, 1297 } 1298 1299 assert.Equal(t, notMatchBoolExpr.GetFields(), []string{"country"}) 1300 fieldToValue["country"] = segutils.CValueEnclosure{ 1301 Dtype: segutils.SS_DT_STRING, 1302 CVal: "Jersey", 1303 } 1304 1305 str, err = notMatchConditionExpr.EvaluateCondition(fieldToValue) 1306 assert.Nil(t, err) 1307 assert.Equal(t, str, "no") 1308 1309 } 1310 1311 func Test_StringExpr(t *testing.T) { 1312 strExpr := 1313 &StringExpr{ 1314 StringExprMode: SEMTextExpr, 1315 TextExpr: &TextExpr{ 1316 IsTerminal: false, 1317 Op: "urldecode", 1318 Value: &StringExpr{ 1319 StringExprMode: SEMRawString, 1320 RawString: "http%3A%2F%2Fwww.splunk.com%2Fdownload%3Fr%3Dheader", 1321 }, 1322 }, 1323 } 1324 assert.Equal(t, strExpr.GetFields(), []string{}) 1325 1326 // Test Evaluate() 1327 fieldToValue := make(map[string]segutils.CValueEnclosure) 1328 1329 value, err := strExpr.Evaluate(fieldToValue) 1330 assert.Nil(t, err) 1331 assert.Equal(t, value, "http://www.splunk.com/download?r=header") 1332 1333 strExpr1 := 1334 &StringExpr{ 1335 StringExprMode: SEMTextExpr, 1336 TextExpr: &TextExpr{ 1337 IsTerminal: false, 1338 Op: "split", 1339 Value: &StringExpr{ 1340 StringExprMode: SEMField, 1341 FieldName: "ident", 1342 }, 1343 Delimiter: &StringExpr{ 1344 StringExprMode: SEMRawString, 1345 RawString: "-", 1346 }, 1347 }, 1348 } 1349 assert.Equal(t, strExpr1.GetFields(), []string{"ident"}) 1350 1351 // Test Evaluate() 1352 fieldToValue["ident"] = segutils.CValueEnclosure{ 1353 Dtype: segutils.SS_DT_STRING, 1354 CVal: "a111d29d-dd70-48b2-8987-a807b4b8bbae", 1355 } 1356 1357 value, err = strExpr1.Evaluate(fieldToValue) 1358 assert.Nil(t, err) 1359 assert.Equal(t, value, "a111d29d dd70 48b2 8987 a807b4b8bbae") 1360 1361 strMax := 1362 &StringExpr{ 1363 StringExprMode: SEMTextExpr, 1364 TextExpr: &TextExpr{ 1365 IsTerminal: false, 1366 Op: "max", 1367 MaxMinValues: []*StringExpr{ 1368 { 1369 StringExprMode: SEMConcatExpr, 1370 ConcatExpr: &ConcatExpr{ 1371 Atoms: []*ConcatAtom{ 1372 {IsField: false, Value: "1"}, 1373 }, 1374 }, 1375 }, 1376 1377 { 1378 StringExprMode: SEMConcatExpr, 1379 ConcatExpr: &ConcatExpr{ 1380 Atoms: []*ConcatAtom{ 1381 {IsField: false, Value: "3"}, 1382 }, 1383 }, 1384 }, 1385 1386 { 1387 StringExprMode: SEMConcatExpr, 1388 ConcatExpr: &ConcatExpr{ 1389 Atoms: []*ConcatAtom{ 1390 {IsField: false, Value: "450"}, 1391 }, 1392 }, 1393 }, 1394 { 1395 1396 StringExprMode: SEMField, 1397 FieldName: "http_status", 1398 }, 1399 }, 1400 }, 1401 } 1402 assert.Equal(t, strMax.GetFields(), []string{"http_status"}) 1403 1404 // Test Evaluate() 1405 fieldToValue["http_status"] = segutils.CValueEnclosure{ 1406 Dtype: segutils.SS_DT_STRING, 1407 CVal: "200", 1408 } 1409 1410 value, err = strMax.Evaluate(fieldToValue) 1411 assert.Nil(t, err) 1412 assert.Equal(t, value, "450") 1413 1414 strMin := 1415 &StringExpr{ 1416 StringExprMode: SEMTextExpr, 1417 TextExpr: &TextExpr{ 1418 IsTerminal: false, 1419 Op: "min", 1420 MaxMinValues: []*StringExpr{ 1421 { 1422 StringExprMode: SEMConcatExpr, 1423 ConcatExpr: &ConcatExpr{ 1424 Atoms: []*ConcatAtom{ 1425 {IsField: false, Value: "1"}, 1426 }, 1427 }, 1428 }, 1429 1430 { 1431 StringExprMode: SEMConcatExpr, 1432 ConcatExpr: &ConcatExpr{ 1433 Atoms: []*ConcatAtom{ 1434 {IsField: false, Value: "3"}, 1435 }, 1436 }, 1437 }, 1438 1439 { 1440 StringExprMode: SEMConcatExpr, 1441 ConcatExpr: &ConcatExpr{ 1442 Atoms: []*ConcatAtom{ 1443 {IsField: false, Value: "450"}, 1444 }, 1445 }, 1446 }, 1447 { 1448 1449 StringExprMode: SEMField, 1450 FieldName: "http_status", 1451 }, 1452 }, 1453 }, 1454 } 1455 assert.Equal(t, strMin.GetFields(), []string{"http_status"}) 1456 1457 // Test Evaluate() 1458 fieldToValue["http_status"] = segutils.CValueEnclosure{ 1459 Dtype: segutils.SS_DT_STRING, 1460 CVal: "200", 1461 } 1462 1463 value, err = strMin.Evaluate(fieldToValue) 1464 assert.Nil(t, err) 1465 assert.Equal(t, value, "1") 1466 1467 strSubStr := &StringExpr{ 1468 StringExprMode: SEMConcatExpr, 1469 ConcatExpr: &ConcatExpr{ 1470 Atoms: []*ConcatAtom{ 1471 { 1472 IsField: false, 1473 TextExpr: &TextExpr{ 1474 IsTerminal: false, 1475 Op: "substr", 1476 Value: &StringExpr{ 1477 StringExprMode: SEMRawString, 1478 RawString: "splendid", 1479 }, 1480 StartIndex: &NumericExpr{ 1481 NumericExprMode: NEMNumber, 1482 IsTerminal: true, 1483 ValueIsField: false, 1484 Value: "1", 1485 }, 1486 LengthExpr: &NumericExpr{ 1487 NumericExprMode: NEMNumber, 1488 IsTerminal: true, 1489 ValueIsField: false, 1490 Value: "3", 1491 }, 1492 }, 1493 }, 1494 { 1495 IsField: false, 1496 TextExpr: &TextExpr{ 1497 IsTerminal: false, 1498 Op: "substr", 1499 Value: &StringExpr{ 1500 StringExprMode: SEMRawString, 1501 RawString: "chunk", 1502 }, 1503 StartIndex: &NumericExpr{ 1504 NumericExprMode: NEMNumber, 1505 IsTerminal: true, 1506 ValueIsField: false, 1507 Value: "-3", 1508 }, 1509 }, 1510 }, 1511 }, 1512 }, 1513 } 1514 assert.Equal(t, strSubStr.GetFields(), []string{}) 1515 1516 value, err = strSubStr.Evaluate(fieldToValue) 1517 assert.Nil(t, err) 1518 assert.Equal(t, value, "splunk") 1519 1520 strToStringBool := 1521 &StringExpr{ 1522 StringExprMode: SEMTextExpr, 1523 TextExpr: &TextExpr{ 1524 IsTerminal: false, 1525 Op: "tostring", 1526 Val: &ValueExpr{ 1527 ValueExprMode: VEMBooleanExpr, 1528 BooleanExpr: &BoolExpr{ 1529 IsTerminal: true, 1530 LeftValue: &ValueExpr{ 1531 ValueExprMode: VEMNumericExpr, 1532 NumericExpr: &NumericExpr{ 1533 NumericExprMode: NEMNumber, 1534 IsTerminal: true, 1535 ValueIsField: false, 1536 Value: "2", 1537 }, 1538 }, 1539 RightValue: &ValueExpr{ 1540 ValueExprMode: VEMNumericExpr, 1541 NumericExpr: &NumericExpr{ 1542 NumericExprMode: NEMNumber, 1543 IsTerminal: true, 1544 ValueIsField: false, 1545 Value: "1", 1546 }, 1547 }, 1548 ValueOp: ">", 1549 }, 1550 }, 1551 }, 1552 } 1553 assert.Equal(t, strToStringBool.GetFields(), []string{}) 1554 1555 value, err = strToStringBool.Evaluate(fieldToValue) 1556 assert.Nil(t, err) 1557 assert.Equal(t, value, "true") 1558 1559 strToStringHex := 1560 &StringExpr{ 1561 StringExprMode: SEMTextExpr, 1562 TextExpr: &TextExpr{ 1563 IsTerminal: false, 1564 Op: "tostring", 1565 Val: &ValueExpr{ 1566 ValueExprMode: VEMNumericExpr, 1567 NumericExpr: &NumericExpr{ 1568 NumericExprMode: NEMNumber, 1569 IsTerminal: true, 1570 ValueIsField: false, 1571 Value: "15", 1572 }, 1573 }, 1574 Format: &StringExpr{ 1575 StringExprMode: SEMRawString, 1576 RawString: "hex", 1577 }, 1578 }, 1579 } 1580 assert.Equal(t, strToStringHex.GetFields(), []string{}) 1581 1582 value, err = strToStringHex.Evaluate(fieldToValue) 1583 assert.Nil(t, err) 1584 assert.Equal(t, value, "0xf") 1585 1586 strToStringCommas := 1587 &StringExpr{ 1588 StringExprMode: SEMTextExpr, 1589 TextExpr: &TextExpr{ 1590 IsTerminal: false, 1591 Op: "tostring", 1592 Val: &ValueExpr{ 1593 ValueExprMode: VEMNumericExpr, 1594 NumericExpr: &NumericExpr{ 1595 NumericExprMode: NEMNumber, 1596 IsTerminal: true, 1597 ValueIsField: false, 1598 Value: "12345.6789", 1599 }, 1600 }, 1601 Format: &StringExpr{ 1602 StringExprMode: SEMRawString, 1603 RawString: "commas", 1604 }, 1605 }, 1606 } 1607 assert.Equal(t, strToStringCommas.GetFields(), []string{}) 1608 1609 value, err = strToStringCommas.Evaluate(fieldToValue) 1610 assert.Nil(t, err) 1611 assert.Equal(t, value, "12,345.68") 1612 1613 strToStringDuration := 1614 &StringExpr{ 1615 StringExprMode: SEMTextExpr, 1616 TextExpr: &TextExpr{ 1617 IsTerminal: false, 1618 Op: "tostring", 1619 Val: &ValueExpr{ 1620 ValueExprMode: VEMNumericExpr, 1621 NumericExpr: &NumericExpr{ 1622 NumericExprMode: NEMNumber, 1623 IsTerminal: true, 1624 ValueIsField: false, 1625 Value: "615", 1626 }, 1627 }, 1628 Format: &StringExpr{ 1629 StringExprMode: SEMRawString, 1630 RawString: "duration", 1631 }, 1632 }, 1633 } 1634 assert.Equal(t, strToStringDuration.GetFields(), []string{}) 1635 1636 value, err = strToStringDuration.Evaluate(fieldToValue) 1637 assert.Nil(t, err) 1638 assert.Equal(t, value, "00:10:15") 1639 1640 } 1641 1642 func Test_RenameExpr(t *testing.T) { 1643 renameToPhrase := &RenameExpr{ 1644 RenameExprMode: REMPhrase, 1645 OriginalPattern: "city", 1646 NewPattern: "test", 1647 } 1648 1649 renameRegex := &RenameExpr{ 1650 RenameExprMode: REMRegex, 1651 OriginalPattern: "app*", 1652 NewPattern: "start*end", 1653 } 1654 1655 renameToExistingField := &RenameExpr{ 1656 RenameExprMode: REMOverride, 1657 OriginalPattern: "http_status", 1658 NewPattern: "", 1659 } 1660 1661 assert.Equal(t, []string{"city"}, renameToPhrase.GetFields()) 1662 assert.Equal(t, []string{}, renameRegex.GetFields()) 1663 assert.Equal(t, []string{"http_status"}, renameToExistingField.GetFields()) 1664 1665 fieldToValue := make(map[string]segutils.CValueEnclosure) 1666 fieldToValue["city"] = segutils.CValueEnclosure{ 1667 Dtype: segutils.SS_DT_STRING, 1668 CVal: "Boston", 1669 } 1670 fieldToValue["http_status"] = segutils.CValueEnclosure{ 1671 Dtype: segutils.SS_DT_STRING, 1672 CVal: "200", 1673 } 1674 1675 val, err := renameToPhrase.Evaluate(fieldToValue, renameToPhrase.GetFields()[0]) 1676 assert.Nil(t, err) 1677 assert.Equal(t, "Boston", val) 1678 1679 val, err = renameToExistingField.Evaluate(fieldToValue, renameToExistingField.GetFields()[0]) 1680 assert.Nil(t, err) 1681 assert.Equal(t, "200", val) 1682 1683 // Test Process Rename Regex logic 1684 // No match column 1685 newCol, err := renameRegex.ProcessRenameRegexExpression("http_status") 1686 assert.Nil(t, err) 1687 assert.Equal(t, "", newCol) 1688 1689 newCol, err = renameRegex.ProcessRenameRegexExpression("app_name") 1690 assert.Nil(t, err) 1691 assert.Equal(t, "start_nameend", newCol) 1692 1693 // Multiple wildcards 1694 renameRegex.OriginalPattern = "ht*_*ta*" 1695 renameRegex.NewPattern = "first*second*third*end" 1696 newCol, err = renameRegex.ProcessRenameRegexExpression("http_status") 1697 1698 assert.Nil(t, err) 1699 assert.Equal(t, newCol, "firsttpsecondsthirdtusend") 1700 1701 // Wrong Pattern 1702 renameRegex.OriginalPattern = "[abc" 1703 renameRegex.NewPattern = "first*second*third*end" 1704 _, err = renameRegex.ProcessRenameRegexExpression("ddd") 1705 1706 assert.NotNil(t, err) 1707 1708 // Test Remove unused GroupByCols by index 1709 bucketResult := &BucketResult{ 1710 GroupByKeys: []string{"http_status", "http_method", "city", "state", "gender", "app_name"}, 1711 BucketKey: []string{"200", "POST", "Boston", "MA", "Male", "sig"}, 1712 } 1713 1714 renameRegex.RemoveBucketResGroupByColumnsByIndex(bucketResult, []int{3, 1, 4}) 1715 assert.Equal(t, []string{"http_status", "city", "app_name"}, bucketResult.GroupByKeys) 1716 assert.Equal(t, []string{"200", "Boston", "sig"}, bucketResult.BucketKey.([]string)) 1717 1718 bucketHolder := &BucketHolder{ 1719 GroupByValues: []string{"200", "POST", "Boston", "MA", "Male", "sig"}, 1720 } 1721 1722 renameRegex.RemoveBucketHolderGroupByColumnsByIndex(bucketHolder, []string{"http_status", "http_method", "city", "state", "gender", "app_name"}, []int{5, 2}) 1723 assert.Equal(t, []string{"200", "POST", "MA", "Male"}, bucketHolder.GroupByValues) 1724 } 1725 1726 func Test_StatisticExpr(t *testing.T) { 1727 1728 statisticExpr := &StatisticExpr{ 1729 StatisticFunctionMode: SFMRare, 1730 Limit: "2", 1731 StatisticOptions: &StatisticOptions{ 1732 CountField: "app_name", 1733 OtherStr: "other", 1734 PercentField: "http_method", 1735 ShowCount: true, 1736 ShowPerc: true, 1737 UseOther: true, 1738 }, 1739 FieldList: []string{"http_method", "weekday"}, 1740 ByClause: []string{"app_name"}, 1741 } 1742 1743 assert.Equal(t, []string{"http_method", "weekday", "app_name"}, statisticExpr.GetGroupByCols()) 1744 1745 bucketResult := &BucketResult{ 1746 ElemCount: 333, 1747 GroupByKeys: []string{"http_method", "weekday", "app_name"}, 1748 BucketKey: []string{"PUT", "Sunday", "sig"}, 1749 } 1750 1751 err := statisticExpr.OverrideGroupByCol(bucketResult, 666) 1752 assert.Nil(t, err) 1753 assert.Equal(t, []string{"50.000000", "Sunday", "333"}, bucketResult.BucketKey.([]string)) 1754 1755 bucketResult1 := &BucketResult{ 1756 ElemCount: 333, 1757 GroupByKeys: []string{"http_method", "http_status", "weekday", "state", "gender", "app_name"}, 1758 BucketKey: []string{"POST", "404", "Sunday", "MA", "Male", "sig"}, 1759 } 1760 1761 bucketResult2 := &BucketResult{ 1762 ElemCount: 111, 1763 GroupByKeys: []string{"http_method", "http_status", "weekday", "state", "gender", "app_name"}, 1764 BucketKey: []string{"Get", "200", "Tuesday", "LA", "Male", "test"}, 1765 } 1766 1767 bucketResult3 := &BucketResult{ 1768 ElemCount: 222, 1769 GroupByKeys: []string{"http_method", "http_status", "weekday", "state", "gender", "app_name"}, 1770 BucketKey: []string{"PUT", "501", "Monday", "NH", "Femali", "sig_test"}, 1771 } 1772 1773 // Test Sorting func. If use the limit option, only the last limit lexicographical of the <field-list> is returned in the search results 1774 results := append(make([]*BucketResult, 0), bucketResult1, bucketResult2, bucketResult3) 1775 err = statisticExpr.SortBucketResult(&results) 1776 assert.Nil(t, err) 1777 assert.Equal(t, 2, len(results)) 1778 assert.Equal(t, bucketResult2, results[0]) 1779 assert.Equal(t, bucketResult1, results[1]) 1780 }