github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/aggregations/segaggs_test.go (about) 1 /* 2 Copyright 2023. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package aggregations 18 19 import ( 20 "encoding/json" 21 "fmt" 22 "math/rand" 23 "strings" 24 "testing" 25 26 "github.com/siglens/siglens/pkg/segment/structs" 27 "github.com/siglens/siglens/pkg/segment/utils" 28 "github.com/stretchr/testify/assert" 29 ) 30 31 type SimpleSearchExpr struct { 32 Op string 33 Field string 34 Values interface{} 35 ValueIsRegex bool 36 ExprType utils.SS_DTYPE 37 DtypeEnclosure *utils.DtypeEnclosure 38 } 39 40 func Test_conditionMatch(t *testing.T) { 41 tests := []struct { 42 name string 43 fieldValue interface{} 44 op string 45 searchValue interface{} 46 expectedBool bool 47 }{ 48 {"EqualStringTrue", "test", "=", "test", true}, 49 {"EqualStringFalse", "test", "=", "fail", false}, 50 {"NotEqualStringTrue", "test", "!=", "fail", true}, 51 {"NotEqualStringFalse", "test", "!=", "test", false}, 52 {"GreaterThanTrue", 10, ">", 5, true}, 53 {"GreaterThanFalse", 3, ">", 5, false}, 54 {"GreaterThanOrEqualTrue", 5, ">=", 5, true}, 55 {"GreaterThanOrEqualFalse", 3, ">=", 5, false}, 56 {"LessThanTrue", 3, "<", 5, true}, 57 {"LessThanFalse", 10, "<", 5, false}, 58 {"LessThanOrEqualTrue", 5, "<=", 5, true}, 59 {"LessThanOrEqualFalse", 10, "<=", 5, false}, 60 {"InvalidOperator", 10, "invalid", 5, false}, 61 {"InvalidFieldValue", "invalid", ">", 5, false}, 62 {"InvalidSearchValue", 5, ">", "invalid", false}, 63 } 64 65 for _, test := range tests { 66 t.Run(test.name, func(t *testing.T) { 67 result := conditionMatch(test.fieldValue, test.op, test.searchValue) 68 assert.Equal(t, test.expectedBool, result) 69 }) 70 } 71 } 72 73 var cities = []string{ 74 "Hyderabad", 75 "New York", 76 "Los Angeles", 77 "Chicago", 78 "Houston", 79 "Phoenix", 80 "Philadelphia", 81 "San Antonio", 82 "San Diego", 83 "Dallas", 84 "San Jose", 85 } 86 87 var countries = []string{ 88 "India", 89 "United States", 90 "Canada", 91 "United Kingdom", 92 "Australia", 93 "Germany", 94 "France", 95 "Spain", 96 "Italy", 97 "Japan", 98 } 99 100 func generateTestRecords(numRecords int) map[string]map[string]interface{} { 101 records := make(map[string]map[string]interface{}, numRecords) 102 103 for i := 0; i < numRecords; i++ { 104 record := make(map[string]interface{}) 105 106 record["timestamp"] = uint64(1659874108987) 107 record["city"] = cities[rand.Intn(len(cities))] 108 record["gender"] = []string{"male", "female"}[rand.Intn(2)] 109 record["country"] = countries[rand.Intn(len(countries))] 110 record["http_method"] = []string{"GET", "POST", "PUT", "DELETE"}[rand.Intn(4)] 111 record["http_status"] = []int64{200, 201, 301, 302, 404}[rand.Intn(5)] 112 record["latitude"] = rand.Float64() * 180 113 record["longitude"] = rand.Float64() * 180 114 115 records[fmt.Sprint(i)] = record 116 } 117 118 return records 119 } 120 121 func getFinalColsForGeneratedTestRecords() map[string]bool { 122 return map[string]bool{"timestamp": true, "city": true, "gender": true, "country": true, "http_method": true, "http_status": true, "latitude": true, "longitude": true} 123 } 124 125 // Test Cases for processTransactionsOnRecords 126 func All_TestCasesForTransactionCommands() (map[int]bool, []*structs.TransactionArguments, map[int]map[string]interface{}) { 127 matchesSomeRecords := make(map[int]bool) 128 searchResults := make(map[int]map[string]interface{}) 129 130 // CASE 1: Only Fields 131 txnArgs1 := &structs.TransactionArguments{ 132 Fields: []string{"gender", "city"}, 133 StartsWith: nil, 134 EndsWith: nil, 135 } 136 matchesSomeRecords[1] = true 137 138 // CASE 2: Only EndsWith 139 txnArgs2 := &structs.TransactionArguments{ 140 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 141 StartsWith: &structs.FilterStringExpr{StringValue: "GET"}, 142 Fields: []string{}, 143 } 144 matchesSomeRecords[2] = true 145 146 // CASE 3: Only StartsWith 147 txnArgs3 := &structs.TransactionArguments{ 148 StartsWith: &structs.FilterStringExpr{StringValue: "GET"}, 149 EndsWith: nil, 150 Fields: []string{}, 151 } 152 matchesSomeRecords[3] = true 153 154 // CASE 4: StartsWith and EndsWith 155 txnArgs4 := &structs.TransactionArguments{ 156 StartsWith: &structs.FilterStringExpr{StringValue: "GET"}, 157 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 158 Fields: []string{}, 159 } 160 matchesSomeRecords[4] = true 161 162 // CASE 5: StartsWith and EndsWith and one Field 163 txnArgs5 := &structs.TransactionArguments{ 164 StartsWith: &structs.FilterStringExpr{StringValue: "GET"}, 165 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 166 Fields: []string{"gender"}, 167 } 168 matchesSomeRecords[5] = true 169 170 // CASE 6: StartsWith and EndsWith and two Fields 171 txnArgs6 := &structs.TransactionArguments{ 172 StartsWith: &structs.FilterStringExpr{StringValue: "GET"}, 173 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 174 Fields: []string{"gender", "country"}, 175 } 176 matchesSomeRecords[6] = true 177 178 // CASE 7: StartsWith and EndsWith with String Clauses only OR: startswith=("GET" OR "POST1") endswith=("DELETE" OR "POST2") 179 txnArgs7 := &structs.TransactionArguments{ 180 StartsWith: &structs.FilterStringExpr{ 181 SearchNode: &structs.ASTNode{ 182 OrFilterCondition: &structs.Condition{ 183 FilterCriteria: []*structs.FilterCriteria{ 184 { 185 MatchFilter: &structs.MatchFilter{ 186 MatchColumn: "*", 187 MatchWords: [][]byte{ 188 []byte("GET"), 189 }, 190 MatchOperator: utils.And, 191 MatchPhrase: []byte("GET"), 192 MatchType: structs.MATCH_PHRASE, 193 }, 194 }, 195 { 196 MatchFilter: &structs.MatchFilter{ 197 MatchColumn: "*", 198 MatchWords: [][]byte{ 199 []byte("POST1"), 200 }, 201 MatchOperator: utils.And, 202 MatchPhrase: []byte("POST1"), 203 MatchType: structs.MATCH_PHRASE, 204 }, 205 }, 206 }, 207 }, 208 }, 209 }, 210 EndsWith: &structs.FilterStringExpr{ 211 SearchNode: &structs.ASTNode{ 212 OrFilterCondition: &structs.Condition{ 213 FilterCriteria: []*structs.FilterCriteria{ 214 { 215 MatchFilter: &structs.MatchFilter{ 216 MatchColumn: "*", 217 MatchWords: [][]byte{ 218 []byte("DELETE"), 219 }, 220 MatchOperator: utils.And, 221 MatchPhrase: []byte("DELETE"), 222 MatchType: structs.MATCH_PHRASE, 223 }, 224 }, 225 { 226 MatchFilter: &structs.MatchFilter{ 227 MatchColumn: "*", 228 MatchWords: [][]byte{ 229 []byte("POST2"), 230 }, 231 MatchOperator: utils.And, 232 MatchPhrase: []byte("POST2"), 233 MatchType: structs.MATCH_PHRASE, 234 }, 235 }, 236 }, 237 }, 238 }, 239 }, 240 Fields: []string{"gender", "country"}, 241 } 242 matchesSomeRecords[7] = true 243 searchResults[7] = map[string]interface{}{ 244 "startswith": [][]*SimpleSearchExpr{ 245 { 246 { 247 Op: "=", 248 Field: "http_method", 249 Values: "GET", 250 ValueIsRegex: false, 251 ExprType: utils.SS_DT_STRING, 252 DtypeEnclosure: &utils.DtypeEnclosure{ 253 Dtype: utils.SS_DT_STRING, 254 StringVal: "GET", 255 }, 256 }, 257 { 258 Op: "=", 259 Field: "http_method", 260 Values: "POST1", 261 ValueIsRegex: false, 262 ExprType: utils.SS_DT_STRING, 263 DtypeEnclosure: &utils.DtypeEnclosure{ 264 Dtype: utils.SS_DT_STRING, 265 StringVal: "POST1", 266 }, 267 }, 268 }, 269 }, 270 "endswith": [][]*SimpleSearchExpr{ 271 { 272 { 273 Op: "=", 274 Field: "http_method", 275 Values: "DELETE", 276 ValueIsRegex: false, 277 ExprType: utils.SS_DT_STRING, 278 DtypeEnclosure: &utils.DtypeEnclosure{ 279 Dtype: utils.SS_DT_STRING, 280 StringVal: "DELETE", 281 }, 282 }, 283 { 284 Op: "=", 285 Field: "http_method", 286 Values: "POST2", 287 ValueIsRegex: false, 288 ExprType: utils.SS_DT_STRING, 289 DtypeEnclosure: &utils.DtypeEnclosure{ 290 Dtype: utils.SS_DT_STRING, 291 StringVal: "POST2", 292 }, 293 }, 294 }, 295 }, 296 } 297 298 // CASE 8: StartsWith and EndsWith with String Clauses only AND (Negative Case): startswith=("GET" AND "POST2") endswith=("POST") 299 txnArgs8 := &structs.TransactionArguments{ 300 StartsWith: &structs.FilterStringExpr{ 301 SearchNode: &structs.ASTNode{ 302 AndFilterCondition: &structs.Condition{ 303 FilterCriteria: []*structs.FilterCriteria{ 304 { 305 MatchFilter: &structs.MatchFilter{ 306 MatchColumn: "*", 307 MatchWords: [][]byte{ 308 []byte("GET"), 309 }, 310 MatchOperator: utils.And, 311 MatchPhrase: []byte("GET"), 312 MatchType: structs.MATCH_PHRASE, 313 }, 314 }, 315 { 316 MatchFilter: &structs.MatchFilter{ 317 MatchColumn: "*", 318 MatchWords: [][]byte{ 319 []byte("POST2"), 320 }, 321 MatchOperator: utils.And, 322 MatchPhrase: []byte("POST2"), 323 MatchType: structs.MATCH_PHRASE, 324 }, 325 }, 326 }, 327 }, 328 }, 329 }, 330 EndsWith: &structs.FilterStringExpr{ 331 SearchNode: &structs.ASTNode{ 332 AndFilterCondition: &structs.Condition{ 333 FilterCriteria: []*structs.FilterCriteria{ 334 { 335 MatchFilter: &structs.MatchFilter{ 336 MatchColumn: "*", 337 MatchWords: [][]byte{ 338 []byte("DELETE"), 339 }, 340 MatchOperator: utils.And, 341 MatchPhrase: []byte("DELETE"), 342 MatchType: structs.MATCH_PHRASE, 343 }, 344 }, 345 }, 346 }, 347 }, 348 }, 349 Fields: []string{"gender", "country"}, 350 } 351 matchesSomeRecords[8] = false 352 353 // CASE 9: StartsWith and EndsWith with String Clauses only AND (Positive Case): startswith=("GET" AND "male") endswith=("DELETE") 354 txnArgs9 := &structs.TransactionArguments{ 355 Fields: []string{"gender", "country"}, 356 StartsWith: &structs.FilterStringExpr{ 357 SearchNode: &structs.ASTNode{ 358 AndFilterCondition: &structs.Condition{ 359 FilterCriteria: []*structs.FilterCriteria{ 360 { 361 MatchFilter: &structs.MatchFilter{ 362 MatchColumn: "*", 363 MatchWords: [][]byte{ 364 []byte("GET"), 365 }, 366 MatchOperator: utils.And, 367 MatchPhrase: []byte("GET"), 368 MatchType: structs.MATCH_PHRASE, 369 }, 370 }, 371 { 372 MatchFilter: &structs.MatchFilter{ 373 MatchColumn: "*", 374 MatchWords: [][]byte{ 375 []byte("male"), 376 }, 377 MatchOperator: utils.And, 378 MatchPhrase: []byte("male"), 379 MatchType: structs.MATCH_PHRASE, 380 }, 381 }, 382 }, 383 }, 384 }, 385 }, 386 EndsWith: &structs.FilterStringExpr{ 387 SearchNode: &structs.ASTNode{ 388 AndFilterCondition: &structs.Condition{ 389 FilterCriteria: []*structs.FilterCriteria{ 390 { 391 MatchFilter: &structs.MatchFilter{ 392 MatchColumn: "*", 393 MatchWords: [][]byte{ 394 []byte("DELETE"), 395 }, 396 MatchOperator: utils.And, 397 MatchPhrase: []byte("DELETE"), 398 MatchType: structs.MATCH_PHRASE, 399 }, 400 }, 401 }, 402 }, 403 }, 404 }, 405 } 406 matchesSomeRecords[9] = true 407 searchResults[9] = map[string]interface{}{ 408 "startswith": [][]*SimpleSearchExpr{ 409 { 410 { 411 Op: "=", 412 Field: "http_method", 413 Values: "GET", 414 ValueIsRegex: false, 415 ExprType: utils.SS_DT_STRING, 416 DtypeEnclosure: &utils.DtypeEnclosure{ 417 Dtype: utils.SS_DT_STRING, 418 StringVal: "GET", 419 StringValBytes: []byte("GET"), 420 }, 421 }, 422 }, 423 { 424 { 425 Op: "=", 426 Field: "gender", 427 Values: "male", 428 ValueIsRegex: false, 429 ExprType: utils.SS_DT_STRING, 430 DtypeEnclosure: &utils.DtypeEnclosure{ 431 Dtype: utils.SS_DT_STRING, 432 StringVal: "male", 433 StringValBytes: []byte("male"), 434 }, 435 }, 436 }, 437 }, 438 "endswith": [][]*SimpleSearchExpr{ 439 { 440 { 441 Op: "=", 442 Field: "http_method", 443 Values: "DELETE", 444 ValueIsRegex: false, 445 ExprType: utils.SS_DT_STRING, 446 DtypeEnclosure: &utils.DtypeEnclosure{ 447 Dtype: utils.SS_DT_STRING, 448 StringVal: "DELETE", 449 }, 450 }, 451 }, 452 }, 453 } 454 455 // CASE 10: StartsWith is a Valid Search Expr and EndsWith is String Value: startswith=status>=300 endswith="DELETE" 456 txnArgs10 := &structs.TransactionArguments{ 457 StartsWith: &structs.FilterStringExpr{ 458 SearchNode: &structs.ASTNode{ 459 AndFilterCondition: &structs.Condition{ 460 FilterCriteria: []*structs.FilterCriteria{ 461 { 462 ExpressionFilter: &structs.ExpressionFilter{ 463 LeftInput: &structs.FilterInput{ 464 Expression: &structs.Expression{ 465 LeftInput: &structs.ExpressionInput{ 466 ColumnValue: nil, 467 ColumnName: "http_status", 468 }, 469 ExpressionOp: utils.Add, 470 RightInput: nil, 471 }, 472 }, 473 FilterOperator: utils.GreaterThanOrEqualTo, 474 RightInput: &structs.FilterInput{ 475 Expression: &structs.Expression{ 476 LeftInput: &structs.ExpressionInput{ 477 ColumnValue: &utils.DtypeEnclosure{ 478 Dtype: utils.SS_DT_UNSIGNED_NUM, 479 UnsignedVal: uint64(300), 480 SignedVal: int64(300), 481 FloatVal: float64(300), 482 StringVal: "300", 483 }, 484 }, 485 ExpressionOp: utils.Add, 486 RightInput: nil, 487 }, 488 }, 489 }, 490 }, 491 }, 492 }, 493 }, 494 }, 495 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 496 } 497 matchesSomeRecords[10] = true 498 searchResults[10] = map[string]interface{}{ 499 "startswith": [][]*SimpleSearchExpr{ 500 { 501 { 502 Op: ">=", 503 Field: "http_status", 504 Values: json.Number("300"), 505 ValueIsRegex: false, 506 ExprType: utils.SS_DT_SIGNED_NUM, 507 DtypeEnclosure: &utils.DtypeEnclosure{ 508 Dtype: utils.SS_DT_SIGNED_NUM, 509 FloatVal: float64(300), 510 UnsignedVal: uint64(300), 511 SignedVal: int64(300), 512 StringVal: "300", 513 }, 514 }, 515 }, 516 }, 517 } 518 519 // CASE 11: StartsWith is not a Valid Search Term (comparing between two string fields) and EndsWith is String value: startswith=city>"Hyderabad" endswith="DELETE" 520 txnArgs11 := &structs.TransactionArguments{ 521 StartsWith: &structs.FilterStringExpr{ 522 SearchNode: &structs.ASTNode{ 523 AndFilterCondition: &structs.Condition{ 524 FilterCriteria: []*structs.FilterCriteria{ 525 { 526 ExpressionFilter: &structs.ExpressionFilter{ 527 LeftInput: &structs.FilterInput{ 528 Expression: &structs.Expression{ 529 LeftInput: &structs.ExpressionInput{ 530 ColumnValue: nil, 531 ColumnName: "city", 532 }, 533 ExpressionOp: utils.Add, 534 RightInput: nil, 535 }, 536 }, 537 FilterOperator: utils.GreaterThan, 538 RightInput: &structs.FilterInput{ 539 Expression: &structs.Expression{ 540 LeftInput: &structs.ExpressionInput{ 541 ColumnValue: &utils.DtypeEnclosure{ 542 Dtype: utils.SS_DT_STRING, 543 StringVal: "Hyderabad", 544 }, 545 }, 546 }, 547 }, 548 }, 549 }, 550 }, 551 }, 552 }, 553 }, 554 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 555 } 556 matchesSomeRecords[11] = false 557 558 // CASE 12: StartsWith is not a Valid Search Term (comparing between string and number fields) and EndsWith is String Clause: startswith=city>300 endswith=("DELETE") 559 txnArgs12 := &structs.TransactionArguments{ 560 StartsWith: &structs.FilterStringExpr{ 561 SearchNode: &structs.ASTNode{ 562 AndFilterCondition: &structs.Condition{ 563 FilterCriteria: []*structs.FilterCriteria{ 564 { 565 ExpressionFilter: &structs.ExpressionFilter{ 566 LeftInput: &structs.FilterInput{ 567 Expression: &structs.Expression{ 568 LeftInput: &structs.ExpressionInput{ 569 ColumnValue: nil, 570 ColumnName: "city", 571 }, 572 ExpressionOp: utils.Add, 573 RightInput: nil, 574 }, 575 }, 576 FilterOperator: utils.GreaterThan, 577 RightInput: &structs.FilterInput{ 578 Expression: &structs.Expression{ 579 LeftInput: &structs.ExpressionInput{ 580 ColumnValue: &utils.DtypeEnclosure{ 581 Dtype: utils.SS_DT_UNSIGNED_NUM, 582 StringVal: "300", 583 UnsignedVal: uint64(300), 584 SignedVal: int64(300), 585 FloatVal: float64(300), 586 }, 587 }, 588 ExpressionOp: utils.Add, 589 RightInput: nil, 590 }, 591 }, 592 }, 593 }, 594 }, 595 }, 596 }, 597 }, 598 EndsWith: &structs.FilterStringExpr{ 599 SearchNode: &structs.ASTNode{ 600 AndFilterCondition: &structs.Condition{ 601 FilterCriteria: []*structs.FilterCriteria{ 602 { 603 MatchFilter: &structs.MatchFilter{ 604 MatchColumn: "*", 605 MatchWords: [][]byte{ 606 []byte("DELETE"), 607 }, 608 MatchOperator: utils.And, 609 MatchPhrase: []byte("DELETE"), 610 MatchType: structs.MATCH_PHRASE, 611 }, 612 }, 613 }, 614 }, 615 }, 616 }, 617 } 618 matchesSomeRecords[12] = false 619 620 // CASE 13: StartsWith is a Valid Search Term (String1 = String2) and EndsWith is String Value: startswith=city="Hyderabad" endswith="DELETE" 621 txnArgs13 := &structs.TransactionArguments{ 622 StartsWith: &structs.FilterStringExpr{ 623 SearchNode: &structs.ASTNode{ 624 AndFilterCondition: &structs.Condition{ 625 FilterCriteria: []*structs.FilterCriteria{ 626 { 627 ExpressionFilter: &structs.ExpressionFilter{ 628 LeftInput: &structs.FilterInput{ 629 Expression: &structs.Expression{ 630 LeftInput: &structs.ExpressionInput{ 631 ColumnValue: nil, 632 ColumnName: "city", 633 }, 634 ExpressionOp: utils.Add, 635 RightInput: nil, 636 }, 637 }, 638 FilterOperator: utils.Equals, 639 RightInput: &structs.FilterInput{ 640 Expression: &structs.Expression{ 641 LeftInput: &structs.ExpressionInput{ 642 ColumnValue: &utils.DtypeEnclosure{ 643 Dtype: utils.SS_DT_STRING, 644 StringVal: "Hyderabad", 645 }, 646 }, 647 }, 648 }, 649 }, 650 }, 651 }, 652 }, 653 }, 654 }, 655 EndsWith: &structs.FilterStringExpr{StringValue: "DELETE"}, 656 } 657 matchesSomeRecords[13] = true 658 searchResults[13] = map[string]interface{}{ 659 "startswith": [][]*SimpleSearchExpr{ 660 { 661 { 662 Op: "=", 663 Field: "city", 664 Values: "Hyderabad", 665 ValueIsRegex: false, 666 ExprType: utils.SS_DT_STRING, 667 DtypeEnclosure: &utils.DtypeEnclosure{ 668 Dtype: utils.SS_DT_STRING, 669 StringVal: "Hyderabad", 670 }, 671 }, 672 }, 673 }, 674 } 675 676 // CASE 14: Eval Expression: transaction gender startswith=eval(status > 300 AND http_method="POST" OR http_method="PUT") 677 txnArgs14 := &structs.TransactionArguments{ 678 Fields: []string{"gender"}, 679 StartsWith: &structs.FilterStringExpr{ 680 EvalBoolExpr: &structs.BoolExpr{ 681 IsTerminal: false, 682 LeftBool: &structs.BoolExpr{ 683 IsTerminal: true, 684 LeftValue: &structs.ValueExpr{ 685 ValueExprMode: structs.VEMNumericExpr, 686 NumericExpr: &structs.NumericExpr{ 687 NumericExprMode: structs.NEMNumberField, 688 IsTerminal: true, 689 ValueIsField: true, 690 Value: "http_status", 691 }, 692 }, 693 RightValue: &structs.ValueExpr{ 694 NumericExpr: &structs.NumericExpr{ 695 NumericExprMode: structs.NEMNumber, 696 IsTerminal: true, 697 ValueIsField: false, 698 Value: "300", 699 }, 700 }, 701 ValueOp: ">", 702 }, 703 RightBool: &structs.BoolExpr{ 704 IsTerminal: false, 705 LeftBool: &structs.BoolExpr{ 706 IsTerminal: true, 707 LeftValue: &structs.ValueExpr{ 708 ValueExprMode: structs.VEMNumericExpr, 709 NumericExpr: &structs.NumericExpr{ 710 NumericExprMode: structs.NEMNumberField, 711 IsTerminal: true, 712 ValueIsField: true, 713 Value: "http_method", 714 }, 715 }, 716 RightValue: &structs.ValueExpr{ 717 ValueExprMode: structs.VEMStringExpr, 718 StringExpr: &structs.StringExpr{ 719 StringExprMode: structs.SEMRawString, 720 RawString: "POST", 721 }, 722 }, 723 ValueOp: "=", 724 }, 725 RightBool: &structs.BoolExpr{ 726 IsTerminal: true, 727 LeftValue: &structs.ValueExpr{ 728 ValueExprMode: structs.VEMNumericExpr, 729 NumericExpr: &structs.NumericExpr{ 730 NumericExprMode: structs.NEMNumberField, 731 IsTerminal: true, 732 ValueIsField: true, 733 Value: "http_method", 734 }, 735 }, 736 RightValue: &structs.ValueExpr{ 737 ValueExprMode: structs.VEMStringExpr, 738 StringExpr: &structs.StringExpr{ 739 StringExprMode: structs.SEMRawString, 740 RawString: "PUT", 741 }, 742 }, 743 ValueOp: "=", 744 }, 745 BoolOp: structs.BoolOpOr, 746 }, 747 BoolOp: structs.BoolOpAnd, 748 }, 749 }, 750 } 751 matchesSomeRecords[14] = true 752 searchResults[14] = map[string]interface{}{ 753 "startswith": [][]*SimpleSearchExpr{ 754 { 755 { 756 Op: ">", 757 Field: "http_status", 758 Values: json.Number("300"), 759 ValueIsRegex: false, 760 ExprType: utils.SS_DT_SIGNED_NUM, 761 DtypeEnclosure: &utils.DtypeEnclosure{ 762 Dtype: utils.SS_DT_SIGNED_NUM, 763 FloatVal: float64(300), 764 UnsignedVal: uint64(300), 765 SignedVal: int64(300), 766 StringVal: "300", 767 }, 768 }, 769 }, 770 { 771 { 772 Op: "=", 773 Field: "http_method", 774 Values: "POST", 775 ValueIsRegex: false, 776 ExprType: utils.SS_DT_STRING, 777 DtypeEnclosure: &utils.DtypeEnclosure{ 778 Dtype: utils.SS_DT_STRING, 779 StringVal: "POST", 780 }, 781 }, 782 { 783 Op: "=", 784 Field: "http_method", 785 Values: "PUT", 786 ValueIsRegex: false, 787 ExprType: utils.SS_DT_STRING, 788 DtypeEnclosure: &utils.DtypeEnclosure{ 789 Dtype: utils.SS_DT_STRING, 790 StringVal: "PUT", 791 }, 792 }, 793 }, 794 }, 795 } 796 797 // CASE 15: String Search Expr: transaction gender startswith="status>300 OR status=201 AND http_method=POST" endswith=eval(status<400) 798 txnArgs15 := &structs.TransactionArguments{ 799 Fields: []string{"gender"}, 800 StartsWith: &structs.FilterStringExpr{ 801 SearchNode: &structs.ASTNode{ 802 AndFilterCondition: &structs.Condition{ 803 FilterCriteria: []*structs.FilterCriteria{ 804 { 805 ExpressionFilter: &structs.ExpressionFilter{ 806 LeftInput: &structs.FilterInput{ 807 Expression: &structs.Expression{ 808 LeftInput: &structs.ExpressionInput{ 809 ColumnValue: nil, 810 ColumnName: "http_method", 811 }, 812 ExpressionOp: utils.Add, 813 RightInput: nil, 814 }, 815 }, 816 RightInput: &structs.FilterInput{ 817 Expression: &structs.Expression{ 818 LeftInput: &structs.ExpressionInput{ 819 ColumnValue: &utils.DtypeEnclosure{ 820 Dtype: utils.SS_DT_STRING, 821 StringVal: "POST", 822 }, 823 ColumnName: "", 824 }, 825 ExpressionOp: utils.Add, 826 RightInput: nil, 827 }, 828 }, 829 FilterOperator: utils.Equals, 830 }, 831 }, 832 }, 833 NestedNodes: []*structs.ASTNode{ 834 { 835 OrFilterCondition: &structs.Condition{ 836 FilterCriteria: []*structs.FilterCriteria{ 837 { 838 ExpressionFilter: &structs.ExpressionFilter{ 839 LeftInput: &structs.FilterInput{ 840 Expression: &structs.Expression{ 841 LeftInput: &structs.ExpressionInput{ 842 ColumnValue: nil, 843 ColumnName: "http_status", 844 }, 845 ExpressionOp: utils.Add, 846 RightInput: nil, 847 }, 848 }, 849 RightInput: &structs.FilterInput{ 850 Expression: &structs.Expression{ 851 LeftInput: &structs.ExpressionInput{ 852 ColumnValue: &utils.DtypeEnclosure{ 853 Dtype: utils.SS_DT_UNSIGNED_NUM, 854 UnsignedVal: uint64(300), 855 SignedVal: int64(300), 856 FloatVal: float64(300), 857 StringVal: "300", 858 }, 859 ColumnName: "", 860 }, 861 ExpressionOp: utils.Add, 862 RightInput: nil, 863 }, 864 }, 865 FilterOperator: utils.GreaterThan, 866 }, 867 }, 868 { 869 ExpressionFilter: &structs.ExpressionFilter{ 870 LeftInput: &structs.FilterInput{ 871 Expression: &structs.Expression{ 872 LeftInput: &structs.ExpressionInput{ 873 ColumnValue: nil, 874 ColumnName: "http_status", 875 }, 876 ExpressionOp: utils.Add, 877 RightInput: nil, 878 }, 879 }, 880 RightInput: &structs.FilterInput{ 881 Expression: &structs.Expression{ 882 LeftInput: &structs.ExpressionInput{ 883 ColumnValue: &utils.DtypeEnclosure{ 884 Dtype: utils.SS_DT_UNSIGNED_NUM, 885 UnsignedVal: uint64(201), 886 SignedVal: int64(201), 887 FloatVal: float64(201), 888 StringVal: "201", 889 }, 890 ColumnName: "", 891 }, 892 ExpressionOp: utils.Add, 893 RightInput: nil, 894 }, 895 }, 896 FilterOperator: utils.Equals, 897 }, 898 }, 899 }, 900 NestedNodes: nil, 901 }, 902 }, 903 }, 904 }, 905 }, 906 }, 907 EndsWith: &structs.FilterStringExpr{ 908 EvalBoolExpr: &structs.BoolExpr{ 909 IsTerminal: true, 910 LeftValue: &structs.ValueExpr{ 911 NumericExpr: &structs.NumericExpr{ 912 IsTerminal: true, 913 NumericExprMode: structs.NEMNumberField, 914 ValueIsField: true, 915 Value: "http_status", 916 }, 917 }, 918 RightValue: &structs.ValueExpr{ 919 NumericExpr: &structs.NumericExpr{ 920 IsTerminal: true, 921 NumericExprMode: structs.NEMNumber, 922 ValueIsField: false, 923 Value: "400", 924 }, 925 }, 926 ValueOp: "<", 927 }, 928 }, 929 } 930 matchesSomeRecords[15] = true 931 searchResults[15] = map[string]interface{}{ 932 "endswith": [][]*SimpleSearchExpr{ 933 { 934 { 935 Op: "<", 936 Field: "http_status", 937 Values: json.Number("400"), 938 ValueIsRegex: false, 939 ExprType: utils.SS_DT_SIGNED_NUM, 940 DtypeEnclosure: &utils.DtypeEnclosure{ 941 Dtype: utils.SS_DT_SIGNED_NUM, 942 FloatVal: float64(400), 943 UnsignedVal: uint64(400), 944 SignedVal: int64(400), 945 StringVal: "400", 946 }, 947 }, 948 }, 949 }, 950 "startswith": [][]*SimpleSearchExpr{ 951 { 952 { 953 Op: ">", 954 Field: "http_status", 955 Values: json.Number("300"), 956 ValueIsRegex: false, 957 ExprType: utils.SS_DT_SIGNED_NUM, 958 DtypeEnclosure: &utils.DtypeEnclosure{ 959 Dtype: utils.SS_DT_SIGNED_NUM, 960 FloatVal: float64(300), 961 UnsignedVal: uint64(300), 962 SignedVal: int64(300), 963 StringVal: "300", 964 }, 965 }, 966 { 967 Op: "=", 968 Field: "http_status", 969 Values: json.Number("201"), 970 ValueIsRegex: false, 971 ExprType: utils.SS_DT_SIGNED_NUM, 972 DtypeEnclosure: &utils.DtypeEnclosure{ 973 Dtype: utils.SS_DT_SIGNED_NUM, 974 FloatVal: float64(201), 975 UnsignedVal: uint64(201), 976 SignedVal: int64(201), 977 StringVal: "201", 978 }, 979 }, 980 }, 981 { 982 { 983 Op: "=", 984 Field: "http_method", 985 Values: "POST", 986 ValueIsRegex: false, 987 ExprType: utils.SS_DT_STRING, 988 DtypeEnclosure: &utils.DtypeEnclosure{ 989 Dtype: utils.SS_DT_STRING, 990 StringVal: "POST", 991 }, 992 }, 993 }, 994 }, 995 } 996 997 // CASE 16: String Search Expr: transaction city startswith="status>300 OR status=201" endswith=eval(status<400) 998 txnArgs16 := &structs.TransactionArguments{ 999 Fields: []string{"city"}, 1000 StartsWith: &structs.FilterStringExpr{ 1001 SearchNode: &structs.ASTNode{ 1002 OrFilterCondition: &structs.Condition{ 1003 FilterCriteria: []*structs.FilterCriteria{ 1004 { 1005 ExpressionFilter: &structs.ExpressionFilter{ 1006 LeftInput: &structs.FilterInput{ 1007 Expression: &structs.Expression{ 1008 LeftInput: &structs.ExpressionInput{ 1009 ColumnValue: nil, 1010 ColumnName: "http_status", 1011 }, 1012 ExpressionOp: utils.Add, 1013 RightInput: nil, 1014 }, 1015 }, 1016 RightInput: &structs.FilterInput{ 1017 Expression: &structs.Expression{ 1018 LeftInput: &structs.ExpressionInput{ 1019 ColumnValue: &utils.DtypeEnclosure{ 1020 Dtype: utils.SS_DT_UNSIGNED_NUM, 1021 UnsignedVal: uint64(300), 1022 SignedVal: int64(300), 1023 FloatVal: float64(300), 1024 StringVal: "300", 1025 }, 1026 ColumnName: "", 1027 }, 1028 ExpressionOp: utils.Add, 1029 RightInput: nil, 1030 }, 1031 }, 1032 FilterOperator: utils.GreaterThan, 1033 }, 1034 }, 1035 { 1036 ExpressionFilter: &structs.ExpressionFilter{ 1037 LeftInput: &structs.FilterInput{ 1038 Expression: &structs.Expression{ 1039 LeftInput: &structs.ExpressionInput{ 1040 ColumnValue: nil, 1041 ColumnName: "http_status", 1042 }, 1043 ExpressionOp: utils.Add, 1044 RightInput: nil, 1045 }, 1046 }, 1047 RightInput: &structs.FilterInput{ 1048 Expression: &structs.Expression{ 1049 LeftInput: &structs.ExpressionInput{ 1050 ColumnValue: &utils.DtypeEnclosure{ 1051 Dtype: utils.SS_DT_UNSIGNED_NUM, 1052 UnsignedVal: uint64(201), 1053 SignedVal: int64(201), 1054 FloatVal: float64(201), 1055 StringVal: "201", 1056 }, 1057 ColumnName: "", 1058 }, 1059 ExpressionOp: utils.Add, 1060 RightInput: nil, 1061 }, 1062 }, 1063 FilterOperator: utils.Equals, 1064 }, 1065 }, 1066 }, 1067 NestedNodes: nil, 1068 }, 1069 }, 1070 }, 1071 EndsWith: &structs.FilterStringExpr{ 1072 EvalBoolExpr: &structs.BoolExpr{ 1073 IsTerminal: true, 1074 LeftValue: &structs.ValueExpr{ 1075 NumericExpr: &structs.NumericExpr{ 1076 IsTerminal: true, 1077 NumericExprMode: structs.NEMNumberField, 1078 ValueIsField: true, 1079 Value: "http_status", 1080 }, 1081 }, 1082 RightValue: &structs.ValueExpr{ 1083 NumericExpr: &structs.NumericExpr{ 1084 IsTerminal: true, 1085 NumericExprMode: structs.NEMNumber, 1086 ValueIsField: false, 1087 Value: "400", 1088 }, 1089 }, 1090 ValueOp: "<", 1091 }, 1092 }, 1093 } 1094 matchesSomeRecords[16] = true 1095 searchResults[16] = map[string]interface{}{ 1096 "endswith": [][]*SimpleSearchExpr{ 1097 { 1098 { 1099 Op: "<", 1100 Field: "http_status", 1101 Values: json.Number("400"), 1102 ValueIsRegex: false, 1103 ExprType: utils.SS_DT_SIGNED_NUM, 1104 DtypeEnclosure: &utils.DtypeEnclosure{ 1105 Dtype: utils.SS_DT_SIGNED_NUM, 1106 FloatVal: float64(400), 1107 UnsignedVal: uint64(400), 1108 SignedVal: int64(400), 1109 StringVal: "400", 1110 }, 1111 }, 1112 }, 1113 }, 1114 "startswith": [][]*SimpleSearchExpr{ 1115 { 1116 { 1117 Op: ">", 1118 Field: "http_status", 1119 Values: json.Number("300"), 1120 ValueIsRegex: false, 1121 ExprType: utils.SS_DT_SIGNED_NUM, 1122 DtypeEnclosure: &utils.DtypeEnclosure{ 1123 Dtype: utils.SS_DT_SIGNED_NUM, 1124 FloatVal: float64(300), 1125 UnsignedVal: uint64(300), 1126 SignedVal: int64(300), 1127 StringVal: "300", 1128 }, 1129 }, 1130 { 1131 Op: "=", 1132 Field: "http_status", 1133 Values: json.Number("201"), 1134 ValueIsRegex: false, 1135 ExprType: utils.SS_DT_SIGNED_NUM, 1136 DtypeEnclosure: &utils.DtypeEnclosure{ 1137 Dtype: utils.SS_DT_SIGNED_NUM, 1138 FloatVal: float64(201), 1139 UnsignedVal: uint64(201), 1140 SignedVal: int64(201), 1141 StringVal: "201", 1142 }, 1143 }, 1144 }, 1145 }, 1146 } 1147 1148 return matchesSomeRecords, []*structs.TransactionArguments{txnArgs1, txnArgs2, txnArgs3, txnArgs4, txnArgs5, txnArgs6, txnArgs7, txnArgs8, txnArgs9, 1149 txnArgs10, txnArgs11, txnArgs12, txnArgs13, txnArgs14, txnArgs15, txnArgs16}, searchResults 1150 } 1151 1152 func Test_processTransactionsOnRecords(t *testing.T) { 1153 1154 allCols := map[string]bool{"city": true, "gender": true} 1155 1156 matchesSomeRecords, allCasesTxnArgs, searchResults := All_TestCasesForTransactionCommands() 1157 1158 for index, txnArgs := range allCasesTxnArgs { 1159 records := generateTestRecords(500) 1160 // Process Transactions 1161 performTransactionCommandRequest(&structs.NodeResult{}, &structs.QueryAggregators{TransactionArguments: txnArgs}, records, allCols, 1, true) 1162 1163 expectedCols := map[string]bool{"duration": true, "event": true, "eventcount": true, "timestamp": true} 1164 1165 for _, field := range txnArgs.Fields { 1166 expectedCols[field] = true 1167 } 1168 1169 assert.Equal(t, expectedCols, allCols) 1170 1171 // Check if the number of records is positive or negative 1172 assert.Equal(t, matchesSomeRecords[index+1], len(records) > 0) 1173 1174 for _, record := range records { 1175 assert.Equal(t, record["timestamp"], uint64(1659874108987)) 1176 assert.Equal(t, record["duration"], uint64(0)) 1177 1178 events := record["event"].([]map[string]interface{}) 1179 1180 initFields := []string{} 1181 1182 for ind, eventMap := range events { 1183 fields := []string{} 1184 1185 for _, field := range txnArgs.Fields { 1186 fields = append(fields, eventMap[field].(string)) 1187 } 1188 1189 // Check if the fields are same for all events by assigning the first event's fields to initFields 1190 if ind == 0 { 1191 initFields = fields 1192 } 1193 1194 assert.Equal(t, fields, initFields) 1195 1196 if txnArgs.StartsWith != nil { 1197 if ind == 0 { 1198 resultData, exists := getResultData(index+1, "startswith", searchResults) 1199 if txnArgs.StartsWith.StringValue != "" { 1200 assert.Equal(t, eventMap["http_method"], txnArgs.StartsWith.StringValue) 1201 } else if txnArgs.StartsWith.EvalBoolExpr != nil { 1202 if exists { 1203 valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr)) 1204 assert.True(t, valid) 1205 } 1206 } else if txnArgs.StartsWith.SearchNode != nil { 1207 if exists { 1208 valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr)) 1209 assert.True(t, valid) 1210 } 1211 } 1212 } 1213 } 1214 1215 if txnArgs.EndsWith != nil { 1216 if ind == len(events)-1 { 1217 resultData, exists := getResultData(index+1, "endswith", searchResults) 1218 if txnArgs.EndsWith.StringValue != "" { 1219 assert.Equal(t, eventMap["http_method"], txnArgs.EndsWith.StringValue) 1220 } else if txnArgs.EndsWith.EvalBoolExpr != nil { 1221 if exists { 1222 valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr)) 1223 assert.True(t, valid) 1224 } 1225 } else if txnArgs.EndsWith.SearchNode != nil { 1226 if exists { 1227 valid := validateSearchExpr(eventMap, resultData.([][]*SimpleSearchExpr)) 1228 assert.True(t, valid) 1229 } 1230 } 1231 } 1232 } 1233 1234 } 1235 } 1236 } 1237 1238 } 1239 1240 func getResultData(resultIndex int, resultType string, resultData map[int]map[string]interface{}) (interface{}, bool) { 1241 resultDataMap, exists := resultData[resultIndex] 1242 if exists { 1243 data, exists := resultDataMap[resultType] 1244 return data, exists 1245 } else { 1246 return nil, false 1247 } 1248 } 1249 1250 func validateSearchString(searchTerm *SimpleSearchExpr, eventMap map[string]interface{}) bool { 1251 fieldValue, exists := eventMap[searchTerm.Field] 1252 if !exists { 1253 return false 1254 } 1255 1256 return conditionMatch(fieldValue, searchTerm.Op, searchTerm.Values) 1257 } 1258 1259 func validateSearchExpr(eventMap map[string]interface{}, resultData [][]*SimpleSearchExpr) bool { 1260 for _, resultAnd := range resultData { 1261 valid := false 1262 for _, resultOr := range resultAnd { 1263 if validateSearchString(resultOr, eventMap) { 1264 valid = true 1265 break 1266 } 1267 } 1268 if !valid { 1269 return false 1270 } 1271 } 1272 return true 1273 } 1274 1275 func Test_performValueColRequestWithoutGroupBy_VEMNumericExpr(t *testing.T) { 1276 1277 // Query 1: * | eval rlatitude=round(latitude, 2) 1278 letColReq := &structs.LetColumnsRequest{ 1279 ValueColRequest: &structs.ValueExpr{ 1280 ValueExprMode: structs.VEMNumericExpr, 1281 NumericExpr: &structs.NumericExpr{ 1282 NumericExprMode: structs.NEMNumericExpr, 1283 Op: "round", 1284 Left: &structs.NumericExpr{ 1285 NumericExprMode: structs.NEMNumberField, 1286 IsTerminal: true, 1287 ValueIsField: true, 1288 Value: "latitude", 1289 }, 1290 Right: &structs.NumericExpr{ 1291 NumericExprMode: structs.NEMNumber, 1292 IsTerminal: true, 1293 ValueIsField: false, 1294 Value: "2", 1295 }, 1296 }, 1297 }, 1298 NewColName: "rlatitude", 1299 } 1300 1301 records := generateTestRecords(500) 1302 finalCols := getFinalColsForGeneratedTestRecords() 1303 1304 // Perform the value column request 1305 err := performValueColRequest(&structs.NodeResult{}, &structs.QueryAggregators{}, letColReq, records, finalCols) 1306 assert.Nil(t, err) 1307 1308 // Check if the new column is added to the records 1309 assert.True(t, finalCols["rlatitude"]) 1310 1311 for _, record := range records { 1312 assert.True(t, record["rlatitude"] != nil) 1313 valueStr := fmt.Sprintf("%.2f", record["latitude"].(float64)) 1314 splitValue := strings.Split(valueStr, ".") 1315 if len(splitValue) > 1 { 1316 assert.Equal(t, len(splitValue[1]), 2) 1317 } 1318 } 1319 1320 } 1321 1322 func Test_performValueColRequestWithoutGroupBy_VEMConditionExpr(t *testing.T) { 1323 // Query: * | eval http_status_mod=if(in(http_status, 400, 500), "Failure", http_status) 1324 letColReq := &structs.LetColumnsRequest{ 1325 ValueColRequest: &structs.ValueExpr{ 1326 ValueExprMode: structs.VEMConditionExpr, 1327 ConditionExpr: &structs.ConditionExpr{ 1328 Op: "if", 1329 BoolExpr: &structs.BoolExpr{ 1330 IsTerminal: true, 1331 LeftValue: &structs.ValueExpr{ 1332 NumericExpr: &structs.NumericExpr{ 1333 NumericExprMode: structs.NEMNumberField, 1334 IsTerminal: true, 1335 ValueIsField: true, 1336 Value: "http_status", 1337 }, 1338 }, 1339 ValueOp: "in", 1340 ValueList: []*structs.ValueExpr{ 1341 { 1342 NumericExpr: &structs.NumericExpr{ 1343 NumericExprMode: structs.NEMNumber, 1344 IsTerminal: true, 1345 ValueIsField: false, 1346 Value: "400", 1347 }, 1348 }, 1349 { 1350 NumericExpr: &structs.NumericExpr{ 1351 NumericExprMode: structs.NEMNumber, 1352 IsTerminal: true, 1353 ValueIsField: false, 1354 Value: "500", 1355 }, 1356 }, 1357 }, 1358 }, 1359 TrueValue: &structs.ValueExpr{ 1360 ValueExprMode: structs.VEMStringExpr, 1361 StringExpr: &structs.StringExpr{ 1362 RawString: "Failure", 1363 }, 1364 }, 1365 FalseValue: &structs.ValueExpr{ 1366 NumericExpr: &structs.NumericExpr{ 1367 NumericExprMode: structs.NEMNumberField, 1368 IsTerminal: true, 1369 ValueIsField: true, 1370 Value: "http_status", 1371 }, 1372 }, 1373 }, 1374 }, 1375 NewColName: "http_status_mod", 1376 } 1377 1378 records := generateTestRecords(500) 1379 finalCols := getFinalColsForGeneratedTestRecords() 1380 1381 // Perform the value column request 1382 err := performValueColRequest(&structs.NodeResult{}, &structs.QueryAggregators{}, letColReq, records, finalCols) 1383 assert.Nil(t, err) 1384 1385 // Check if the new column is added to the records 1386 assert.True(t, finalCols["http_status_mod"]) 1387 1388 for _, record := range records { 1389 assert.True(t, record["http_status_mod"] != nil) 1390 httpStatus := record["http_status"].(int64) 1391 if httpStatus == 400 || httpStatus == 500 { 1392 assert.Equal(t, "Failure", record["http_status_mod"]) 1393 } else { 1394 assert.Equal(t, fmt.Sprint(httpStatus), record["http_status_mod"]) 1395 } 1396 } 1397 } 1398 1399 func Test_performValueColRequestWithoutGroupBy_VEMStringExpr(t *testing.T) { 1400 // Query: * | eval country_city=country.":-".city 1401 letColReq := &structs.LetColumnsRequest{ 1402 ValueColRequest: &structs.ValueExpr{ 1403 ValueExprMode: structs.VEMStringExpr, 1404 StringExpr: &structs.StringExpr{ 1405 StringExprMode: structs.SEMConcatExpr, 1406 ConcatExpr: &structs.ConcatExpr{ 1407 Atoms: []*structs.ConcatAtom{ 1408 { 1409 IsField: true, 1410 Value: "country", 1411 }, 1412 { 1413 IsField: false, 1414 Value: ":-", 1415 }, 1416 { 1417 IsField: true, 1418 Value: "city", 1419 }, 1420 }, 1421 }, 1422 }, 1423 }, 1424 NewColName: "country_city", 1425 } 1426 1427 records := generateTestRecords(500) 1428 finalCols := getFinalColsForGeneratedTestRecords() 1429 1430 // Perform the value column request 1431 err := performValueColRequest(&structs.NodeResult{}, &structs.QueryAggregators{}, letColReq, records, finalCols) 1432 1433 assert.Nil(t, err) 1434 1435 // Check if the new column is added to the records 1436 assert.True(t, finalCols["country_city"]) 1437 1438 for _, record := range records { 1439 assert.True(t, record["country_city"] != nil) 1440 country := record["country"].(string) 1441 city := record["city"].(string) 1442 assert.Equal(t, country+":-"+city, record["country_city"]) 1443 } 1444 } 1445 1446 func Test_getColumnsToKeepAndRemove(t *testing.T) { 1447 tests := []struct { 1448 name string 1449 cols []string 1450 wildcardCols []string 1451 keepMatches bool 1452 wantIndices []int 1453 wantColsToKeep []string 1454 wantColsToRemove []string 1455 }{ 1456 { 1457 name: "No wildcards, keepMatches true", 1458 cols: []string{"id", "name", "email"}, 1459 wildcardCols: []string{}, 1460 keepMatches: true, 1461 wantIndices: []int{}, 1462 wantColsToKeep: []string{}, 1463 wantColsToRemove: []string{"id", "name", "email"}, 1464 }, 1465 { 1466 name: "No wildcards, keepMatches false", 1467 cols: []string{"id", "name", "email"}, 1468 wildcardCols: []string{}, 1469 keepMatches: false, 1470 wantIndices: []int{0, 1, 2}, 1471 wantColsToKeep: []string{"id", "name", "email"}, 1472 wantColsToRemove: []string{}, 1473 }, 1474 { 1475 name: "Exact match one wildcard, keepMatches true", 1476 cols: []string{"id", "name", "email"}, 1477 wildcardCols: []string{"name"}, 1478 keepMatches: true, 1479 wantIndices: []int{1}, 1480 wantColsToKeep: []string{"name"}, 1481 wantColsToRemove: []string{"id", "email"}, 1482 }, 1483 { 1484 name: "Wildcard matches multiple columns, keepMatches true", 1485 cols: []string{"user_id", "username", "user_email", "age"}, 1486 wildcardCols: []string{"user_*"}, 1487 keepMatches: true, 1488 wantIndices: []int{0, 2}, 1489 wantColsToKeep: []string{"user_id", "user_email"}, 1490 wantColsToRemove: []string{"username", "age"}, 1491 }, 1492 { 1493 name: "Wildcard matches none, keepMatches false", 1494 cols: []string{"id", "name", "email"}, 1495 wildcardCols: []string{"user_*"}, 1496 keepMatches: false, 1497 wantIndices: []int{0, 1, 2}, 1498 wantColsToKeep: []string{"id", "name", "email"}, 1499 wantColsToRemove: []string{}, 1500 }, 1501 { 1502 name: "Multiple wildcards with overlaps, keepMatches true", 1503 cols: []string{"user_id", "admin_id", "username", "email"}, 1504 wildcardCols: []string{"user_*", "*_id"}, 1505 keepMatches: true, 1506 wantIndices: []int{0, 1}, 1507 wantColsToKeep: []string{"user_id", "admin_id"}, 1508 wantColsToRemove: []string{"username", "email"}, 1509 }, 1510 { 1511 name: "Empty cols, keepMatches true", 1512 cols: []string{}, 1513 wildcardCols: []string{"user_*"}, 1514 keepMatches: true, 1515 wantIndices: []int{}, 1516 wantColsToKeep: []string{}, 1517 wantColsToRemove: []string{}, 1518 }, 1519 { 1520 name: "Wildcard matches all, keepMatches false", 1521 cols: []string{"user_id", "user_name", "user_email"}, 1522 wildcardCols: []string{"user_*"}, 1523 keepMatches: false, 1524 wantIndices: []int{}, 1525 wantColsToKeep: []string{}, 1526 wantColsToRemove: []string{"user_id", "user_name", "user_email"}, 1527 }, 1528 { 1529 name: "Complex wildcards, partial matches", 1530 cols: []string{"user_id", "admin_id", "username", "user_profile", "user_email", "age"}, 1531 wildcardCols: []string{"user_*", "*name"}, 1532 keepMatches: true, 1533 wantIndices: []int{0, 2, 3, 4}, 1534 wantColsToKeep: []string{"user_id", "username", "user_profile", "user_email"}, 1535 wantColsToRemove: []string{"admin_id", "age"}, 1536 }, 1537 } 1538 1539 for _, tt := range tests { 1540 t.Run(tt.name, func(t *testing.T) { 1541 gotIndices, gotColsToKeep, gotColsToRemove := getColumnsToKeepAndRemove(tt.cols, tt.wildcardCols, tt.keepMatches) 1542 1543 assert.Equal(t, tt.wantIndices, gotIndices) 1544 assert.Equal(t, tt.wantColsToKeep, gotColsToKeep) 1545 assert.Equal(t, tt.wantColsToRemove, gotColsToRemove) 1546 }) 1547 } 1548 }