github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/segment/structs/evaluationstructs.go (about) 1 /* 2 Copyright 2023. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package structs 18 19 import ( 20 "bytes" 21 "fmt" 22 "math" 23 "net" 24 "net/url" 25 "regexp" 26 "sort" 27 "strconv" 28 "strings" 29 "sync" 30 "time" 31 32 "github.com/axiomhq/hyperloglog" 33 "github.com/dustin/go-humanize" 34 35 "github.com/siglens/siglens/pkg/segment/utils" 36 ) 37 38 // These structs are used to organize boolean, string, and numeric expressions. 39 // The expressions can contain fields. To evaluate one of these expressions, 40 // use GetFields() to find all the fields used directly or indirectly by the 41 // expression, then create a fieldToValue map specifying the value for each of 42 // these fields, and call Evaluate(fieldToValue). 43 type BoolExpr struct { 44 IsTerminal bool 45 46 // Only used when IsTerminal is true. 47 LeftValue *ValueExpr 48 RightValue *ValueExpr 49 ValueOp string // Only = or != for strings; can also be <, <=, >, >= for numbers. 50 ValueList []*ValueExpr //Use for in(<value>, <list>) 51 52 // Only used when IsTerminal is false. For a unary BoolOp, RightExpr should be nil. 53 LeftBool *BoolExpr 54 RightBool *BoolExpr 55 BoolOp BoolOperator 56 } 57 58 type RenameExpr struct { 59 RenameExprMode RenameExprMode 60 OriginalPattern string 61 NewPattern string 62 } 63 64 type RexExpr struct { 65 Pattern string 66 FieldName string 67 RexColNames []string 68 } 69 70 type StatisticExpr struct { 71 StatisticFunctionMode StatisticFunctionMode 72 Limit string 73 StatisticOptions *StatisticOptions 74 FieldList []string //Must have FieldList 75 ByClause []string 76 } 77 78 type StatisticOptions struct { 79 CountField string 80 OtherStr string 81 PercentField string 82 ShowCount bool 83 ShowPerc bool 84 UseOther bool 85 } 86 87 type DedupExpr struct { 88 Limit uint64 89 FieldList []string // Must have FieldList 90 DedupOptions *DedupOptions 91 DedupSortEles []*SortElement 92 DedupSortAscending []int // Derived from DedupSortEles.SortByAsc values. 93 94 // DedupCombinations maps combinations to a map mapping the record index 95 // (of all included records for this combination) to the sort values for 96 // that record. For example, if Limit is 3, each inner map will have at 97 // most 3 entries and will store the index and sort values of the top 3 98 // records for that combination. 99 DedupCombinations map[string]map[int][]SortValue 100 PrevCombination string 101 102 DedupRecords map[string]map[string]interface{} 103 NumProcessedSegments uint64 104 ProcessedSegmentsLock sync.Mutex 105 } 106 107 type DedupOptions struct { 108 Consecutive bool 109 KeepEmpty bool 110 KeepEvents bool 111 } 112 113 type SortExpr struct { 114 SortEles []*SortElement 115 // Limit int // to be finished 116 117 SortAscending []int 118 SortRecords map[string]map[string]interface{} 119 NumProcessedSegments uint64 120 ProcessedSegmentsLock sync.Mutex 121 } 122 123 type SortElement struct { 124 SortByAsc bool 125 Op string 126 Field string 127 } 128 129 type SortValue struct { 130 Val string 131 InterpretAs string // Should be "ip", "num", "str", "auto", or "" 132 } 133 134 // See ValueExprMode type definition for which fields are valid for each mode. 135 type ValueExpr struct { 136 ValueExprMode ValueExprMode 137 138 FloatValue float64 139 NumericExpr *NumericExpr 140 StringExpr *StringExpr 141 ConditionExpr *ConditionExpr 142 BooleanExpr *BoolExpr 143 } 144 145 type ConcatExpr struct { 146 Atoms []*ConcatAtom 147 } 148 149 type ConcatAtom struct { 150 IsField bool 151 Value string 152 TextExpr *TextExpr 153 } 154 155 type NumericExpr struct { 156 NumericExprMode NumericExprMode 157 158 IsTerminal bool 159 160 // Only used when IsTerminal is true. 161 ValueIsField bool 162 Value string 163 164 // Only used when IsTerminal is false. 165 Op string // Either +, -, /, *, abs, ceil, round, sqrt, len 166 Left *NumericExpr 167 Right *NumericExpr 168 Val *StringExpr 169 } 170 171 type StringExpr struct { 172 StringExprMode StringExprMode 173 RawString string // only used when mode is RawString 174 FieldName string // only used when mode is Field 175 ConcatExpr *ConcatExpr // only used when mode is Concat 176 TextExpr *TextExpr // only used when mode is TextExpr 177 } 178 179 type TextExpr struct { 180 IsTerminal bool 181 Op string //lower, ltrim, rtrim 182 Value *StringExpr 183 StrToRemove string 184 Delimiter *StringExpr 185 MaxMinValues []*StringExpr 186 StartIndex *NumericExpr 187 LengthExpr *NumericExpr 188 Val *ValueExpr 189 Format *StringExpr 190 } 191 192 type ConditionExpr struct { 193 Op string //if 194 BoolExpr *BoolExpr 195 TrueValue *ValueExpr //if bool expr is true, take this value 196 FalseValue *ValueExpr 197 } 198 199 type TimechartExpr struct { 200 TcOptions *TcOptions 201 BinOptions *BinOptions 202 SingleAgg *SingleAgg 203 ByField string // group by this field inside each time range bucket (timechart) 204 LimitExpr *LimitExpr 205 } 206 207 type LimitExpr struct { 208 IsTop bool // True: keeps the N highest scoring distinct values of the split-by field 209 Num int 210 LimitScoreMode LimitScoreMode 211 } 212 213 type SingleAgg struct { 214 MeasureOperations []*MeasureAggregator 215 //Split By clause 216 } 217 218 type TcOptions struct { 219 BinOptions *BinOptions 220 UseNull bool 221 UseOther bool 222 NullStr string 223 OtherStr string 224 } 225 226 type BinOptions struct { 227 SpanOptions *SpanOptions 228 } 229 230 type SpanOptions struct { 231 DefaultSettings bool 232 SpanLength *SpanLength 233 } 234 235 type SpanLength struct { 236 Num int 237 TimeScalr utils.TimeUnit 238 } 239 240 type SplitByClause struct { 241 Field string 242 TcOptions *TcOptions 243 // Where clause: to be finished 244 } 245 246 // This structure is used to store values which are not within limit. And These values will be merged into the 'other' category. 247 type TMLimitResult struct { 248 ValIsInLimit map[string]bool 249 GroupValScoreMap map[string]*utils.CValueEnclosure 250 Hll *hyperloglog.Sketch 251 StrSet map[string]struct{} 252 OtherCValArr []*utils.CValueEnclosure 253 } 254 255 type BoolOperator uint8 256 257 const ( 258 BoolOpNot BoolOperator = iota 259 BoolOpAnd 260 BoolOpOr 261 ) 262 263 type StatisticFunctionMode uint8 264 265 const ( 266 SFMTop = iota 267 SFMRare 268 ) 269 270 type RenameExprMode uint8 271 272 const ( 273 REMPhrase = iota //Rename with a phrase 274 REMRegex //Rename fields with similar names using a wildcard 275 REMOverride //Rename to a existing field 276 ) 277 278 type LimitScoreMode uint8 279 280 const ( 281 LSMBySum = iota // If only a single aggregation is specified, the score is based on the sum of the values in the aggregation 282 LSMByFreq // Otherwise the score is based on the frequency of the by field's val 283 ) 284 285 type ValueExprMode uint8 286 287 const ( 288 VEMNumericExpr = iota // Only NumricExpr is valid 289 VEMStringExpr // Only StringExpr is valid 290 VEMConditionExpr // Only ConditionExpr is valud 291 VEMBooleanExpr // Only BooleanExpr is valid 292 ) 293 294 type StringExprMode uint8 295 296 const ( 297 SEMRawString = iota // only used when mode is RawString 298 SEMField // only used when mode is Field 299 SEMConcatExpr // only used when mode is Concat 300 SEMTextExpr // only used when mode is TextExpr 301 ) 302 303 type NumericExprMode uint8 304 305 const ( 306 NEMNumber = iota // only used when mode is a Number 307 NEMLenString // only used when mode is a str (used for len()) 308 NEMNumberField // only used when mode is Field (Field can be evaluated to a float) 309 NEMLenField // only used when mode is Field (Field can not be evaluated to a float, used for len()) 310 NEMNumericExpr // only used when mode is a NumericExpr 311 ) 312 313 // Evaluate this BoolExpr to a boolean, replacing each field in the expression 314 // with the value specified by fieldToValue. Each field listed by GetFields() 315 // must be in fieldToValue. 316 func (self *BoolExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (bool, error) { 317 if self.IsTerminal { 318 if self.ValueOp == "in" { 319 inFlag, err := isInValueList(fieldToValue, self.LeftValue, self.ValueList) 320 if err != nil { 321 return false, fmt.Errorf("BoolExpr.Evaluate: can not evaluate Eval In function: %v", err) 322 } 323 return inFlag, err 324 } else if self.ValueOp == "isbool" { 325 val, err := self.LeftValue.EvaluateToString(fieldToValue) 326 if err != nil { 327 return false, fmt.Errorf("isbool: can not evaluate to String: %v", err) 328 } 329 isBool := strings.ToLower(val) == "true" || strings.ToLower(val) == "false" || val == "0" || val == "1" 330 return isBool, nil 331 332 } else if self.ValueOp == "isint" { 333 val, err := self.LeftValue.EvaluateToString(fieldToValue) 334 if err != nil { 335 return false, err 336 } 337 338 _, parseErr := strconv.Atoi(val) 339 return parseErr == nil, nil 340 } else if self.ValueOp == "isstr" { 341 _, floatErr := self.LeftValue.EvaluateToFloat(fieldToValue) 342 343 if floatErr == nil { 344 return false, nil 345 } 346 347 _, strErr := self.LeftValue.EvaluateToString(fieldToValue) 348 return strErr == nil, nil 349 } else if self.ValueOp == "isnull" { 350 // Get the fields associated with this expression 351 fields := self.GetFields() 352 if len(fields) == 0 { 353 return false, fmt.Errorf("BoolExpr.Evaluate: No fields found for isnull operation") 354 } 355 356 // Check the first field's value in the fieldToValue map 357 value, exists := fieldToValue[fields[0]] 358 if !exists { 359 return false, fmt.Errorf("BoolExpr.Evaluate: Field '%s' not found in data", fields[0]) 360 } 361 // Check if the value's Dtype is SS_DT_BACKFILL 362 if value.Dtype == utils.SS_DT_BACKFILL { 363 return true, nil 364 } 365 return false, nil 366 } else if self.ValueOp == "like" { 367 leftStr, errLeftStr := self.LeftValue.EvaluateToString(fieldToValue) 368 if errLeftStr != nil { 369 return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating left side of LIKE to string: %v", errLeftStr) 370 } 371 372 rightStr, errRightStr := self.RightValue.EvaluateToString(fieldToValue) 373 if errRightStr != nil { 374 return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating right side of LIKE to string: %v", errRightStr) 375 } 376 377 regexPattern := strings.Replace(strings.Replace(regexp.QuoteMeta(rightStr), "%", ".*", -1), "_", ".", -1) 378 matched, err := regexp.MatchString("^"+regexPattern+"$", leftStr) 379 if err != nil { 380 return false, fmt.Errorf("BoolExpr.Evaluate: regex error in LIKE operation: %v", err) 381 } 382 return matched, nil 383 } else if self.ValueOp == "match" { 384 leftStr, errLeftStr := self.LeftValue.EvaluateToString(fieldToValue) 385 if errLeftStr != nil { 386 return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating left side of MATCH to string: %v", errLeftStr) 387 } 388 389 rightStr, errRightStr := self.RightValue.EvaluateToString(fieldToValue) 390 if errRightStr != nil { 391 return false, fmt.Errorf("BoolExpr.Evaluate: error evaluating right side of MATCH to string: %v", errRightStr) 392 } 393 394 matched, err := regexp.MatchString(rightStr, leftStr) 395 if err != nil { 396 return false, fmt.Errorf("BoolExpr.Evaluate: regex error in MATCH operation: %v", err) 397 } 398 return matched, nil 399 400 } else if self.ValueOp == "cidrmatch" { 401 cidrStr, errCidr := self.LeftValue.EvaluateToString(fieldToValue) 402 ipStr, errIp := self.RightValue.EvaluateToString(fieldToValue) 403 if errCidr != nil || errIp != nil { 404 return false, fmt.Errorf("cidrmatch: error evaluating arguments: %v, %v", errCidr, errIp) 405 } 406 407 match, err := isIPInCIDR(cidrStr, ipStr) 408 if err != nil { 409 return false, fmt.Errorf("cidrmatch: error in matching CIDR: %v", err) 410 } 411 return match, nil 412 } 413 414 leftStr, errLeftStr := self.LeftValue.EvaluateToString(fieldToValue) 415 rightStr, errRightStr := self.RightValue.EvaluateToString(fieldToValue) 416 leftFloat, errLeftFloat := self.LeftValue.EvaluateToFloat(fieldToValue) 417 rightFloat, errRightFloat := self.RightValue.EvaluateToFloat(fieldToValue) 418 419 if errLeftFloat == nil && errRightFloat == nil { 420 switch self.ValueOp { 421 case "=": 422 return leftFloat == rightFloat, nil 423 case "!=": 424 return leftFloat != rightFloat, nil 425 case "<": 426 return leftFloat < rightFloat, nil 427 case ">": 428 return leftFloat > rightFloat, nil 429 case "<=": 430 return leftFloat <= rightFloat, nil 431 case ">=": 432 return leftFloat >= rightFloat, nil 433 default: 434 return false, fmt.Errorf("BoolExpr.Evaluate: invalid ValueOp %v for floats", self.ValueOp) 435 } 436 } else if errLeftStr == nil && errRightStr == nil { 437 switch self.ValueOp { 438 case "=": 439 return leftStr == rightStr, nil 440 case "!=": 441 442 return leftStr != rightStr, nil 443 default: 444 return false, fmt.Errorf("BoolExpr.Evaluate: invalid ValueOp %v for strings", self.ValueOp) 445 } 446 } else { 447 if errLeftStr != nil && errLeftFloat != nil { 448 return false, fmt.Errorf("BoolExpr.Evaluate: left cannot be evaluated to a string or float") 449 } 450 if errRightStr != nil && errRightFloat != nil { 451 return false, fmt.Errorf("BoolExpr.Evaluate: right cannot be evaluated to a string or float") 452 } 453 454 return false, fmt.Errorf("BoolExpr.Evaluate: left and right ValueExpr have different types") 455 } 456 } else { // IsTerminal is false 457 left, err := self.LeftBool.Evaluate(fieldToValue) 458 if err != nil { 459 return false, err 460 } 461 462 var right bool 463 if self.RightBool != nil { 464 var err error 465 right, err = self.RightBool.Evaluate(fieldToValue) 466 if err != nil { 467 return false, err 468 } 469 } 470 471 switch self.BoolOp { 472 case BoolOpNot: 473 return !left, nil 474 case BoolOpAnd: 475 return left && right, nil 476 case BoolOpOr: 477 return left || right, nil 478 default: 479 return false, fmt.Errorf("invalid BoolOp: %v", self.BoolOp) 480 } 481 } 482 } 483 484 func isIPInCIDR(cidrStr, ipStr string) (bool, error) { 485 _, cidrNet, err := net.ParseCIDR(cidrStr) 486 if err != nil { 487 return false, err 488 } 489 ip := net.ParseIP(ipStr) 490 if ip == nil { 491 return false, fmt.Errorf("invalid IP address") 492 } 493 494 return cidrNet.Contains(ip), nil 495 } 496 497 func isInValueList(fieldToValue map[string]utils.CValueEnclosure, value *ValueExpr, valueList []*ValueExpr) (bool, error) { 498 valueStr, err := value.EvaluateToString(fieldToValue) 499 if err != nil { 500 return false, fmt.Errorf("isInValueList: can not evaluate to String: %v", err) 501 } 502 503 for _, atom := range valueList { 504 atomStr, err := atom.EvaluateToString(fieldToValue) 505 if err != nil { 506 return false, fmt.Errorf("isInValueList: can not evaluate to String: %v", err) 507 } 508 509 if atomStr == valueStr { 510 return true, nil 511 } 512 } 513 514 return false, nil 515 } 516 517 func (self *BoolExpr) GetFields() []string { 518 if self.IsTerminal { 519 fields := make([]string, 0) 520 521 if self.RightValue != nil { 522 fields = append(fields, self.LeftValue.GetFields()...) 523 fields = append(fields, self.RightValue.GetFields()...) 524 } else { 525 fields = append(fields, self.LeftValue.GetFields()...) 526 } 527 528 //Append fields from the InExpr value list 529 for _, ValueExpr := range self.ValueList { 530 fields = append(fields, ValueExpr.GetFields()...) 531 } 532 return fields 533 } else { 534 // When IsTerminal is false, LeftBool must not be nil, but RightBool will be 535 // nil if BoolOp is a unary operation. 536 if self.RightBool == nil { 537 return self.LeftBool.GetFields() 538 } 539 540 return append(self.LeftBool.GetFields(), self.RightBool.GetFields()...) 541 } 542 } 543 544 // Try evaluating this ValueExpr to a string value, replacing each field in the 545 // expression with the value specified by fieldToValue. Each field listed by 546 // GetFields() must be in fieldToValue. 547 // 548 // A ValueExpr can be evaluated to a string or float, so if this fails you may 549 // want to call ValueExpr.EvaluateToFloat(). 550 func (self *ValueExpr) EvaluateToString(fieldToValue map[string]utils.CValueEnclosure) (string, error) { 551 switch self.ValueExprMode { 552 case VEMStringExpr: 553 str, err := self.StringExpr.Evaluate(fieldToValue) 554 if err != nil { 555 return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to string %v", err) 556 } 557 return str, nil 558 //In this case, field name will be recognized as part of NumericExpr at first. It it can not be converted to float64, it should be evaluated as a str 559 case VEMNumericExpr: 560 floatValue, err := self.NumericExpr.Evaluate(fieldToValue) 561 if err != nil { 562 //Because parsing is successful and it can not evaluate as a float in here, 563 //There is one possibility: field name may not be float 564 str, err := getValueAsString(fieldToValue, self.NumericExpr.Value) 565 566 if err == nil { 567 return str, nil 568 } 569 570 return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to float64 or string: %v", err) 571 } 572 return strconv.FormatFloat(floatValue, 'f', -1, 64), nil 573 case VEMConditionExpr: 574 str, err := self.ConditionExpr.EvaluateCondition(fieldToValue) 575 if err != nil { 576 return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to string %v", err) 577 } 578 return str, nil 579 case VEMBooleanExpr: 580 boolResult, err := self.BooleanExpr.Evaluate(fieldToValue) 581 if err != nil { 582 return "", err 583 } 584 return strconv.FormatBool(boolResult), nil 585 default: 586 return "", fmt.Errorf("ValueExpr.EvaluateToString: cannot evaluate to string") 587 } 588 } 589 590 func (self *StringExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (string, error) { 591 switch self.StringExprMode { 592 case SEMRawString: 593 return self.RawString, nil 594 case SEMField: 595 if floatValue, err := getValueAsFloat(fieldToValue, self.FieldName); err == nil { 596 return strconv.FormatFloat(floatValue, 'f', -1, 64), nil 597 } 598 599 if str, err := getValueAsString(fieldToValue, self.FieldName); err == nil { 600 return str, nil 601 } 602 return "", fmt.Errorf("StringExpr.Evaluate: cannot evaluate to field") 603 case SEMConcatExpr: 604 return self.ConcatExpr.Evaluate(fieldToValue) 605 case SEMTextExpr: 606 return self.TextExpr.EvaluateText(fieldToValue) 607 default: 608 return "", fmt.Errorf("StringExpr.Evaluate: cannot evaluate to string") 609 } 610 } 611 612 // Try evaluating this ValueExpr to a float value, replacing each field in the 613 // expression with the value specified by fieldToValue. Each field listed by 614 // GetFields() must be in fieldToValue. 615 // 616 // A ValueExpr can be evaluated to a string or float, so if this fails you may 617 // want to call ValueExpr.EvaluateToString(). 618 func (self *ValueExpr) EvaluateToFloat(fieldToValue map[string]utils.CValueEnclosure) (float64, error) { 619 switch self.ValueExprMode { 620 case VEMNumericExpr: 621 return self.NumericExpr.Evaluate(fieldToValue) 622 default: 623 return 0, fmt.Errorf("ValueExpr.EvaluateToFloat: cannot evaluate to float") 624 } 625 } 626 627 func (self *ValueExpr) GetFields() []string { 628 switch self.ValueExprMode { 629 case VEMNumericExpr: 630 return self.NumericExpr.GetFields() 631 case VEMStringExpr: 632 return self.StringExpr.GetFields() 633 case VEMConditionExpr: 634 return self.ConditionExpr.GetFields() 635 case VEMBooleanExpr: 636 return self.BooleanExpr.GetFields() 637 default: 638 return []string{} 639 } 640 } 641 642 func (self *RexExpr) GetFields() []string { 643 var fields []string 644 fields = append(fields, self.FieldName) 645 return fields 646 } 647 648 func (self *RenameExpr) GetFields() []string { 649 fields := make([]string, 0) 650 651 switch self.RenameExprMode { 652 case REMPhrase: 653 fallthrough 654 case REMOverride: 655 fields = append(fields, self.OriginalPattern) 656 return fields 657 default: 658 return []string{} 659 } 660 } 661 662 func (self *StringExpr) GetFields() []string { 663 switch self.StringExprMode { 664 case SEMConcatExpr: 665 return self.ConcatExpr.GetFields() 666 case SEMTextExpr: 667 return self.TextExpr.GetFields() 668 case SEMField: 669 return []string{self.FieldName} 670 default: 671 return []string{} 672 } 673 } 674 675 // Concatenate all the atoms in this ConcatExpr, replacing all fields with the 676 // values specified by fieldToValue. Each field listed by GetFields() must be in 677 // fieldToValue. 678 func (self *ConcatExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (string, error) { 679 result := "" 680 for _, atom := range self.Atoms { 681 if atom.IsField { 682 value, err := getValueAsString(fieldToValue, atom.Value) 683 if err != nil { 684 return "", err 685 } 686 687 result += value 688 } else if atom.TextExpr != nil { 689 value, err := atom.TextExpr.EvaluateText(fieldToValue) 690 if err != nil { 691 return "", err 692 } 693 result += value 694 } else { 695 result += atom.Value 696 } 697 } 698 699 return result, nil 700 } 701 702 func (self *ConcatExpr) GetFields() []string { 703 fields := make([]string, 0) 704 for _, atom := range self.Atoms { 705 if atom.IsField { 706 fields = append(fields, atom.Value) 707 } else if atom.TextExpr != nil { 708 textFields := atom.TextExpr.GetFields() 709 if len(textFields) > 0 { 710 fields = append(fields, textFields...) 711 } 712 } 713 } 714 715 return fields 716 } 717 718 func GetBucketKey(BucketKey interface{}, keyIndex int) string { 719 switch bucketKey := BucketKey.(type) { 720 case []string: 721 return bucketKey[keyIndex] 722 case string: 723 return bucketKey 724 default: 725 return "" 726 } 727 } 728 729 func (self *StatisticExpr) OverrideGroupByCol(bucketResult *BucketResult, resTotal uint64) error { 730 731 cellValueStr := "" 732 for keyIndex, groupByCol := range bucketResult.GroupByKeys { 733 if !self.StatisticOptions.ShowCount || !self.StatisticOptions.ShowPerc || (self.StatisticOptions.CountField != groupByCol && self.StatisticOptions.PercentField != groupByCol) { 734 continue 735 } 736 737 if self.StatisticOptions.ShowCount && self.StatisticOptions.CountField == groupByCol { 738 cellValueStr = strconv.FormatUint(bucketResult.ElemCount, 10) 739 } 740 741 if self.StatisticOptions.ShowPerc && self.StatisticOptions.PercentField == groupByCol { 742 percent := float64(bucketResult.ElemCount) / float64(resTotal) * 100 743 cellValueStr = fmt.Sprintf("%.6f", percent) 744 } 745 746 // Set the appropriate element of BucketKey to cellValueStr. 747 switch bucketKey := bucketResult.BucketKey.(type) { 748 case []string: 749 bucketKey[keyIndex] = cellValueStr 750 bucketResult.BucketKey = bucketKey 751 case string: 752 if keyIndex != 0 { 753 return fmt.Errorf("OverrideGroupByCol: expected keyIndex to be 0, not %v", keyIndex) 754 } 755 bucketResult.BucketKey = cellValueStr 756 default: 757 return fmt.Errorf("OverrideGroupByCol: bucket key has unexpected type: %T", bucketKey) 758 } 759 } 760 return nil 761 } 762 763 func (self *StatisticExpr) SetCountToStatRes(statRes map[string]utils.CValueEnclosure, elemCount uint64) { 764 statRes[self.StatisticOptions.CountField] = utils.CValueEnclosure{ 765 Dtype: utils.SS_DT_UNSIGNED_NUM, 766 CVal: elemCount, 767 } 768 } 769 770 func (self *StatisticExpr) SetPercToStatRes(statRes map[string]utils.CValueEnclosure, elemCount uint64, resTotal uint64) { 771 percent := float64(elemCount) / float64(resTotal) * 100 772 statRes[self.StatisticOptions.PercentField] = utils.CValueEnclosure{ 773 Dtype: utils.SS_DT_STRING, 774 CVal: fmt.Sprintf("%.6f", percent), 775 } 776 } 777 778 func (self *StatisticExpr) sortByBucketKey(a, b *BucketResult, fieldList []string, fieldToGroupByKeyIndex map[string]int) bool { 779 780 for _, field := range fieldList { 781 keyIndex := fieldToGroupByKeyIndex[field] 782 if GetBucketKey(a.BucketKey, keyIndex) < GetBucketKey(b.BucketKey, keyIndex) { 783 return false 784 } else if GetBucketKey(a.BucketKey, keyIndex) > GetBucketKey(b.BucketKey, keyIndex) { 785 return true 786 } 787 } 788 return true 789 } 790 791 func (self *StatisticExpr) SortBucketResult(results *[]*BucketResult) error { 792 793 //GroupByKeys -> BucketKey 794 fieldToGroupByKeyIndex := make(map[string]int, len(self.FieldList)) 795 796 //If use the limit option, only the last limit lexicographical of the <field-list> is returned in the search results 797 //Therefore, we should sort the bucket by lexicographical value and retain a limited number of values 798 if len(self.Limit) > 0 { 799 800 for index, groupByKey := range (*results)[0].GroupByKeys { 801 for _, field := range self.FieldList { 802 if field == groupByKey { 803 fieldToGroupByKeyIndex[field] = index 804 } 805 } 806 } 807 808 //Moving the if statement outside of the sorting process can reduce the number of conditional checks 809 //Sort results based on the lexicographical value of their field list 810 switch self.StatisticFunctionMode { 811 case SFMTop: 812 sort.Slice(*results, func(index1, index2 int) bool { 813 return self.sortByBucketKey((*results)[index1], (*results)[index2], self.FieldList, fieldToGroupByKeyIndex) 814 }) 815 case SFMRare: 816 sort.Slice(*results, func(index1, index2 int) bool { 817 return !self.sortByBucketKey((*results)[index1], (*results)[index2], self.FieldList, fieldToGroupByKeyIndex) 818 }) 819 } 820 821 limit, err := strconv.Atoi(self.Limit) 822 if err != nil { 823 return fmt.Errorf("SortBucketResult: cannot convert %v to int", self.Limit) 824 } 825 826 // Only return unique limit field combinations 827 // Since the results are already in order, and there is no Set in Go, we can use a string arr to record the previous field combinations 828 // If the current field combination is different from the previous one, it means we have finished processing data for one field combination (we need to process limit combinations in total, limit is a number) 829 uniqueFieldsCombination := make([]string, len(self.FieldList)) 830 combinationCount := 0 831 newResults := make([]*BucketResult, 0) 832 for _, bucketResult := range *results { 833 combinationExist := true 834 for index, fieldName := range self.FieldList { 835 keyIndex := fieldToGroupByKeyIndex[fieldName] 836 val := GetBucketKey(bucketResult.BucketKey, keyIndex) 837 if uniqueFieldsCombination[index] != val { 838 uniqueFieldsCombination[index] = val 839 combinationExist = false 840 } 841 842 statEnclosure, exists := bucketResult.StatRes[fieldName] 843 statVal, err := statEnclosure.GetString() 844 if exists && err == nil && uniqueFieldsCombination[index] != statVal { 845 uniqueFieldsCombination[index] = val 846 combinationExist = false 847 } 848 } 849 850 // If there is a stats groupby block before statistic groupby block. E.g. ... | stats count BY http_status, gender | rare 1 http_status, 851 // In this case, each http_status will be divided by two genders, so we should merge them into one row here 852 //Fields combination does not exist 853 if !combinationExist { 854 combinationCount++ 855 if combinationCount > limit { 856 *results = newResults 857 return nil 858 } 859 newResults = append(newResults, bucketResult) 860 } else { 861 newResults[combinationCount-1].ElemCount += bucketResult.ElemCount 862 } 863 } 864 865 } else { //No limit option, sort results by its values frequency 866 switch self.StatisticFunctionMode { 867 case SFMTop: 868 sort.Slice(*results, func(index1, index2 int) bool { 869 return (*results)[index1].ElemCount > (*results)[index2].ElemCount 870 }) 871 case SFMRare: 872 sort.Slice(*results, func(index1, index2 int) bool { 873 return (*results)[index1].ElemCount < (*results)[index2].ElemCount 874 }) 875 } 876 } 877 878 return nil 879 } 880 881 // Only display fields which in StatisticExpr 882 func (self *StatisticExpr) RemoveFieldsNotInExprForBucketRes(bucketResult *BucketResult) error { 883 groupByCols := self.GetGroupByCols() 884 groupByKeys := make([]string, 0) 885 bucketKey := make([]string, 0) 886 switch bucketResult.BucketKey.(type) { 887 case []string: 888 bucketKeyStrs := bucketResult.BucketKey.([]string) 889 890 for _, field := range groupByCols { 891 for rowIndex, groupByCol := range bucketResult.GroupByKeys { 892 if field == groupByCol { 893 groupByKeys = append(groupByKeys, field) 894 bucketKey = append(bucketKey, bucketKeyStrs[rowIndex]) 895 break 896 } 897 //Can not find field in GroupByCol, so it may in the StatRes 898 val, exists := bucketResult.StatRes[field] 899 if exists { 900 valStr, _ := val.GetString() 901 groupByKeys = append(groupByKeys, field) 902 bucketKey = append(bucketKey, valStr) 903 delete(bucketResult.StatRes, field) 904 } 905 } 906 } 907 bucketResult.BucketKey = bucketKey 908 case string: 909 if len(groupByCols) == 0 { 910 bucketResult.BucketKey = nil 911 } else { 912 groupByKeys = groupByCols 913 // The GroupByCols of the Statistic block increase, so the new columns must come from the Stats function 914 if len(groupByCols) > 1 { 915 newBucketKey := make([]string, 0) 916 for i := 0; i < len(groupByCols); i++ { 917 val, exists := bucketResult.StatRes[groupByCols[i]] 918 if exists { 919 str, err := val.GetString() 920 if err != nil { 921 return fmt.Errorf("RemoveFieldsNotInExpr: %v", err) 922 } 923 newBucketKey = append(newBucketKey, str) 924 } else { 925 newBucketKey = append(newBucketKey, bucketResult.BucketKey.(string)) 926 } 927 } 928 bucketResult.BucketKey = newBucketKey 929 } 930 } 931 default: 932 return fmt.Errorf("RemoveFieldsNotInExpr: bucket key has unexpected type: %T", bucketKey) 933 } 934 935 // Remove unused func in stats res 936 for statColName := range bucketResult.StatRes { 937 statColInGroupByCols := false 938 for _, groupByCol := range groupByCols { 939 if groupByCol == statColName { 940 statColInGroupByCols = true 941 break 942 } 943 } 944 if !statColInGroupByCols { 945 delete(bucketResult.StatRes, statColName) 946 } 947 } 948 949 bucketResult.GroupByKeys = groupByKeys 950 return nil 951 } 952 953 func (self *StatisticExpr) GetGroupByCols() []string { 954 return append(self.FieldList, self.ByClause...) 955 } 956 957 func (self *RenameExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure, fieldName string) (string, error) { 958 return getValueAsString(fieldToValue, fieldName) 959 } 960 961 func (self *RexExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure, rexExp *regexp.Regexp) (map[string]string, error) { 962 963 fieldValue, err := getValueAsString(fieldToValue, self.FieldName) 964 if err != nil { 965 return nil, fmt.Errorf("RexExpr.Evaluate: %v", err) 966 } 967 968 return MatchAndExtractGroups(fieldValue, rexExp) 969 } 970 971 func MatchAndExtractGroups(str string, rexExp *regexp.Regexp) (map[string]string, error) { 972 match := rexExp.FindStringSubmatch(str) 973 if len(match) == 0 { 974 return nil, fmt.Errorf("MatchAndExtractGroups: no str in field match the pattern") 975 } 976 if len(rexExp.SubexpNames()) == 0 { 977 return nil, fmt.Errorf("MatchAndExtractGroups: no field create from the pattern") 978 } 979 980 result := make(map[string]string) 981 for i, name := range rexExp.SubexpNames() { 982 if i != 0 && name != "" { 983 result[name] = match[i] 984 } 985 } 986 987 return result, nil 988 } 989 990 // Check if colName match the OriginalPattern 991 func (self *RenameExpr) CheckIfMatch(colName string) bool { 992 regexPattern := `\b` + strings.ReplaceAll(self.OriginalPattern, "*", "(.*)") + `\b` 993 regex, err := regexp.Compile(regexPattern) 994 if err != nil { 995 return false 996 } 997 998 matchingParts := regex.FindStringSubmatch(colName) 999 return len(matchingParts) != 0 1000 } 1001 1002 // Check if colName matches the specified pattern and replace wildcards to generate a new colName. 1003 func (self *RenameExpr) ProcessRenameRegexExpression(colName string) (string, error) { 1004 1005 originalPattern := self.OriginalPattern 1006 newPattern := self.NewPattern 1007 1008 regexPattern := `\b` + strings.ReplaceAll(originalPattern, "*", "(.*)") + `\b` 1009 regex, err := regexp.Compile(regexPattern) 1010 if err != nil { 1011 return "", fmt.Errorf("ProcessRenameRegexExpression: There are some errors in the pattern: %v", err) 1012 } 1013 1014 matchingParts := regex.FindStringSubmatch(colName) 1015 if len(matchingParts) == 0 { 1016 return "", nil 1017 } 1018 1019 result := newPattern 1020 for _, match := range matchingParts[1:] { 1021 result = strings.Replace(result, "*", match, 1) 1022 } 1023 1024 return result, nil 1025 } 1026 1027 func (self *RenameExpr) RemoveColsByIndex(strs []string, indexToRemove []int) []string { 1028 results := make([]string, 0) 1029 1030 for index, val := range strs { 1031 shouldRemove := false 1032 for _, delIndex := range indexToRemove { 1033 if delIndex == index { 1034 shouldRemove = true 1035 break 1036 } 1037 } 1038 if shouldRemove { 1039 continue 1040 } 1041 results = append(results, val) 1042 } 1043 return results 1044 } 1045 1046 func (self *RenameExpr) RemoveBucketResGroupByColumnsByIndex(bucketResult *BucketResult, indexToRemove []int) { 1047 1048 if len(indexToRemove) == 0 { 1049 return 1050 } 1051 1052 bucketResult.GroupByKeys = self.RemoveColsByIndex(bucketResult.GroupByKeys, indexToRemove) 1053 1054 switch bucketKey := bucketResult.BucketKey.(type) { 1055 case []string: 1056 bucketResult.BucketKey = self.RemoveColsByIndex(bucketKey, indexToRemove) 1057 case string: 1058 bucketResult.BucketKey = nil 1059 } 1060 1061 } 1062 1063 // Remove unused GroupByVals in Bucket Holder 1064 func (self *RenameExpr) RemoveBucketHolderGroupByColumnsByIndex(bucketHolder *BucketHolder, groupByCols []string, indexToRemove []int) { 1065 1066 if len(indexToRemove) == 0 { 1067 return 1068 } 1069 1070 groupByVals := make([]string, 0) 1071 for index := range groupByCols { 1072 shouldRemove := false 1073 for _, delIndex := range indexToRemove { 1074 if delIndex == index { 1075 shouldRemove = true 1076 break 1077 } 1078 } 1079 if shouldRemove { 1080 continue 1081 } 1082 groupByVals = append(groupByVals, bucketHolder.GroupByValues[index]) 1083 } 1084 1085 bucketHolder.GroupByValues = groupByVals 1086 1087 } 1088 1089 // Evaluate this NumericExpr to a float, replacing each field in the expression 1090 // with the value specified by fieldToValue. Each field listed by GetFields() 1091 // must be in fieldToValue. 1092 func (self *NumericExpr) Evaluate(fieldToValue map[string]utils.CValueEnclosure) (float64, error) { 1093 if self.Op == "now" { 1094 timestamp := time.Now().Unix() 1095 return float64(timestamp), nil 1096 } 1097 if self.IsTerminal { 1098 if self.ValueIsField { 1099 switch self.NumericExprMode { 1100 case NEMNumberField: 1101 return getValueAsFloat(fieldToValue, self.Value) 1102 case NEMLenField: 1103 return float64(len(fieldToValue[self.Value].CVal.(string))), nil 1104 } 1105 } else { 1106 switch self.NumericExprMode { 1107 case NEMNumber: 1108 value, err := strconv.ParseFloat(self.Value, 64) 1109 if err != nil { 1110 return 0, fmt.Errorf("NumericExpr.Evaluate: cannot convert %v to float", self.Value) 1111 } 1112 return value, nil 1113 case NEMLenString: 1114 value := float64(len(self.Value)) 1115 return value, nil 1116 } 1117 } 1118 return 0, fmt.Errorf("NumericExpr.Evaluate: cannot convert %v to float", self.Value) 1119 } else { 1120 1121 left := float64(0) 1122 var err error 1123 if self.Left != nil { 1124 left, err = self.Left.Evaluate(fieldToValue) 1125 if err != nil { 1126 1127 return 0, err 1128 } 1129 } 1130 1131 var right float64 1132 if self.Right != nil { 1133 right, err = self.Right.Evaluate(fieldToValue) 1134 if err != nil { 1135 return 0, err 1136 } 1137 } 1138 1139 switch self.Op { 1140 case "+": 1141 return left + right, nil 1142 case "-": 1143 return left - right, nil 1144 case "*": 1145 return left * right, nil 1146 case "/": 1147 return left / right, nil 1148 case "abs": 1149 return math.Abs(left), nil 1150 case "ceil": 1151 return math.Ceil(left), nil 1152 case "round": 1153 if self.Right != nil { 1154 return round(left, int(right)), nil 1155 } else { 1156 return math.Round(left), nil 1157 } 1158 case "sqrt": 1159 if left < 0 { 1160 return -1, fmt.Errorf("NumericExpr.Evaluate: Negative values cannot be converted to square roots: %v", left) 1161 } 1162 return math.Sqrt(left), nil 1163 case "len": 1164 return left, nil 1165 case "exact": 1166 result, err := self.Left.Evaluate(fieldToValue) 1167 if err != nil { 1168 return 0, err 1169 } 1170 return result, nil 1171 case "exp": 1172 exp, err := self.Left.Evaluate(fieldToValue) 1173 if err != nil { 1174 return 0, err 1175 } 1176 return math.Exp(exp), nil 1177 case "tonumber": 1178 if self.Val == nil { 1179 return 0, fmt.Errorf("NumericExpr.Evaluate: tonumber operation requires a string expression") 1180 } 1181 strValue, err := self.Val.Evaluate(fieldToValue) 1182 if err != nil { 1183 return 0, fmt.Errorf("NumericExpr.Evaluate: Error in tonumber operation: %v", err) 1184 } 1185 base := 10 1186 if self.Right != nil { 1187 baseValue, err := self.Right.Evaluate(fieldToValue) 1188 if err != nil { 1189 return 0, err 1190 } 1191 base = int(baseValue) 1192 if base < 2 || base > 36 { 1193 return 0, fmt.Errorf("NumericExpr.Evaluate: Invalid base for tonumber: %v", base) 1194 } 1195 } 1196 number, err := strconv.ParseInt(strValue, base, 64) 1197 if err != nil { 1198 return 0, fmt.Errorf("NumericExpr.Evaluate: cannot convert '%v' to number with base %d", strValue, base) 1199 } 1200 return float64(number), nil 1201 1202 default: 1203 return 0, fmt.Errorf("NumericExpr.Evaluate: unexpected operation: %v", self.Op) 1204 } 1205 } 1206 } 1207 1208 func (self *TextExpr) EvaluateText(fieldToValue map[string]utils.CValueEnclosure) (string, error) { 1209 if self.Op == "max" { 1210 if len(self.MaxMinValues) == 0 { 1211 return "", fmt.Errorf("TextExpr.Evaluate: no values provided for 'max' operation") 1212 } 1213 maxString := "" 1214 for _, expr := range self.MaxMinValues { 1215 result, err := expr.Evaluate(fieldToValue) 1216 if err != nil { 1217 return "", err 1218 } 1219 if result > maxString { 1220 maxString = result 1221 } 1222 } 1223 return maxString, nil 1224 1225 } else if self.Op == "min" { 1226 if len(self.MaxMinValues) == 0 { 1227 return "", fmt.Errorf("TextExpr.Evaluate: no values provided for 'min' operation") 1228 } 1229 minString := "" 1230 for _, expr := range self.MaxMinValues { 1231 result, err := expr.Evaluate(fieldToValue) 1232 if err != nil { 1233 return "", err 1234 } 1235 if minString == "" || result < minString { 1236 minString = result 1237 } 1238 } 1239 return minString, nil 1240 1241 } else if self.Op == "tostring" { 1242 valueStr, err := self.Val.EvaluateToString(fieldToValue) 1243 if err != nil { 1244 return "", fmt.Errorf("TextExpr.Evaluate: failed to evaluate value for 'tostring' operation: %v", err) 1245 } 1246 if self.Format != nil { 1247 formatStr, err := self.Format.Evaluate(fieldToValue) 1248 if err != nil { 1249 return "", fmt.Errorf("TextExpr.Evaluate: failed to evaluate format for 'tostring' operation: %v", err) 1250 } 1251 switch formatStr { 1252 case "hex": 1253 num, convErr := strconv.Atoi(valueStr) 1254 if convErr != nil { 1255 return "", fmt.Errorf("TextExpr.Evaluate: failed to convert value '%s' to integer for hex formatting: %v", valueStr, convErr) 1256 } 1257 return fmt.Sprintf("%#x", num), nil 1258 case "commas": 1259 num, convErr := strconv.ParseFloat(valueStr, 64) 1260 if convErr != nil { 1261 return "", fmt.Errorf("TextExpr.Evaluate: failed to convert value '%s' to float for comma formatting: %v", valueStr, convErr) 1262 } 1263 roundedNum := math.Round(num*100) / 100 1264 formattedNum := humanize.CommafWithDigits(roundedNum, 2) 1265 return formattedNum, nil 1266 case "duration": 1267 num, convErr := strconv.Atoi(valueStr) 1268 if convErr != nil { 1269 return "", fmt.Errorf("TextExpr.Evaluate: failed to convert value '%s' to seconds for duration formatting: %v", valueStr, convErr) 1270 } 1271 hours := num / 3600 1272 minutes := (num % 3600) / 60 1273 seconds := num % 60 1274 return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds), nil 1275 default: 1276 return "", fmt.Errorf("TextExpr.Evaluate: unsupported format '%s' for tostring operation", formatStr) 1277 } 1278 } else { 1279 return valueStr, nil 1280 } 1281 } 1282 cellValueStr, err := self.Value.Evaluate(fieldToValue) 1283 if err != nil { 1284 return "", fmt.Errorf("TextExpr.Evaluate: can not evaluate text as a str: %v", err) 1285 } 1286 1287 switch self.Op { 1288 case "lower": 1289 return strings.ToLower(cellValueStr), nil 1290 case "ltrim": 1291 return strings.TrimLeft(cellValueStr, self.StrToRemove), nil 1292 case "rtrim": 1293 return strings.TrimRight(cellValueStr, self.StrToRemove), nil 1294 case "urldecode": 1295 decodedStr, decodeErr := url.QueryUnescape(cellValueStr) 1296 if decodeErr != nil { 1297 return "", fmt.Errorf("TextExpr.Evaluate: failed to decode URL: %v", decodeErr) 1298 } 1299 return decodedStr, nil 1300 1301 case "split": 1302 delimiterStr, err := self.Delimiter.Evaluate(fieldToValue) 1303 if err != nil { 1304 return "", fmt.Errorf("TextExpr.Evaluate: cannot evaluate delimiter as a string: %v", err) 1305 } 1306 1307 return strings.Join(strings.Split(cellValueStr, delimiterStr), " "), nil 1308 1309 case "substr": 1310 baseString, err := self.Value.Evaluate(fieldToValue) 1311 if err != nil { 1312 return "", err 1313 } 1314 startIndexFloat, err := self.StartIndex.Evaluate(fieldToValue) 1315 if err != nil { 1316 return "", err 1317 } 1318 startIndex := int(startIndexFloat) 1319 if startIndex > 0 { 1320 startIndex = startIndex - 1 1321 } 1322 if startIndex < 0 { 1323 startIndex = len(baseString) + startIndex 1324 } 1325 if startIndex < 0 || startIndex >= len(baseString) { 1326 return "", fmt.Errorf("substr: start index is out of range") 1327 } 1328 substrLength := len(baseString) - startIndex 1329 if self.LengthExpr != nil { 1330 lengthFloat, err := self.LengthExpr.Evaluate(fieldToValue) 1331 if err != nil { 1332 return "", err 1333 } 1334 substrLength = int(lengthFloat) 1335 if substrLength < 0 || startIndex+substrLength > len(baseString) { 1336 return "", fmt.Errorf("substr: length leads to out of range substring") 1337 } 1338 } 1339 endIndex := startIndex + substrLength 1340 if endIndex > len(baseString) { 1341 endIndex = len(baseString) 1342 } 1343 return baseString[startIndex:endIndex], nil 1344 1345 default: 1346 return "", fmt.Errorf("TextExpr.Evaluate: unexpected operation: %v", self.Op) 1347 } 1348 } 1349 1350 // In this case, if we can not evaluate numeric expr to a float, we should evaluate it as a str 1351 func (self *ValueExpr) EvaluateValueExprAsString(fieldToValue map[string]utils.CValueEnclosure) (string, error) { 1352 var str string 1353 var err error 1354 switch self.ValueExprMode { 1355 case VEMNumericExpr: 1356 floatValue, err := self.EvaluateToFloat(fieldToValue) 1357 str = fmt.Sprintf("%v", floatValue) 1358 if err != nil { 1359 str, err = self.EvaluateToString(fieldToValue) 1360 if err != nil { 1361 return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate to a ValueExpr: %v", err) 1362 } 1363 } 1364 case VEMStringExpr: 1365 str, err = self.EvaluateToString(fieldToValue) 1366 if err != nil { 1367 return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate to a ValueExpr: %v", err) 1368 } 1369 } 1370 return str, nil 1371 } 1372 1373 // Field may come from BoolExpr or ValueExpr 1374 func (self *ConditionExpr) EvaluateCondition(fieldToValue map[string]utils.CValueEnclosure) (string, error) { 1375 predicateFlag, err := self.BoolExpr.Evaluate(fieldToValue) 1376 if err != nil { 1377 return "", fmt.Errorf("ConditionExpr.Evaluate: %v", err) 1378 } 1379 1380 trueValue, err := self.TrueValue.EvaluateValueExprAsString(fieldToValue) 1381 if err != nil { 1382 return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate trueValue to a ValueExpr: %v", err) 1383 } 1384 falseValue, err := self.FalseValue.EvaluateValueExprAsString(fieldToValue) 1385 if err != nil { 1386 return "", fmt.Errorf("ConditionExpr.Evaluate: can not evaluate falseValue to a ValueExpr: %v", err) 1387 } 1388 1389 switch self.Op { 1390 case "if": 1391 if predicateFlag { 1392 return trueValue, nil 1393 } else { 1394 return falseValue, nil 1395 } 1396 default: 1397 return "", fmt.Errorf("ConditionExpr.Evaluate: unexpected operation: %v", self.Op) 1398 } 1399 1400 } 1401 1402 func (self *TextExpr) GetFields() []string { 1403 fields := make([]string, 0) 1404 if self.IsTerminal || (self.Op != "max" && self.Op != "min") { 1405 if self.Value != nil { 1406 fields = append(fields, self.Value.GetFields()...) 1407 } 1408 if self.Val != nil { 1409 fields = append(fields, self.Val.GetFields()...) 1410 } 1411 if self.Delimiter != nil { 1412 fields = append(fields, self.Delimiter.GetFields()...) 1413 } 1414 if self.StartIndex != nil { 1415 fields = append(fields, self.StartIndex.GetFields()...) 1416 } 1417 if self.LengthExpr != nil { 1418 fields = append(fields, self.LengthExpr.GetFields()...) 1419 } 1420 if self.Format != nil { 1421 fields = append(fields, self.Format.GetFields()...) 1422 } 1423 return fields 1424 } 1425 for _, expr := range self.MaxMinValues { 1426 fields = append(fields, expr.GetFields()...) 1427 } 1428 return fields 1429 1430 } 1431 1432 // Append all the fields in ConditionExpr 1433 func (self *ConditionExpr) GetFields() []string { 1434 fields := make([]string, 0) 1435 fields = append(fields, self.BoolExpr.GetFields()...) 1436 fields = append(fields, self.TrueValue.GetFields()...) 1437 fields = append(fields, self.FalseValue.GetFields()...) 1438 return fields 1439 } 1440 1441 // Specifying a value and a precision 1442 func round(number float64, precision int) float64 { 1443 scale := math.Pow10(precision) 1444 return math.Round(number*scale) / scale 1445 } 1446 1447 func (self *NumericExpr) GetFields() []string { 1448 fields := make([]string, 0) 1449 if self.Val != nil { 1450 return append(fields, self.Val.GetFields()...) 1451 } 1452 if self.IsTerminal { 1453 if self.Op == "now" { 1454 return fields 1455 } 1456 if self.ValueIsField { 1457 return []string{self.Value} 1458 } else { 1459 return []string{} 1460 } 1461 } else if self.Right != nil { 1462 return append(self.Left.GetFields(), self.Right.GetFields()...) 1463 } else { 1464 return self.Left.GetFields() 1465 } 1466 } 1467 1468 func getValueAsString(fieldToValue map[string]utils.CValueEnclosure, field string) (string, error) { 1469 enclosure, ok := fieldToValue[field] 1470 if !ok { 1471 return "", fmt.Errorf("Missing field %v", field) 1472 } 1473 1474 return enclosure.GetString() 1475 } 1476 1477 func getValueAsFloat(fieldToValue map[string]utils.CValueEnclosure, field string) (float64, error) { 1478 enclosure, ok := fieldToValue[field] 1479 if !ok { 1480 return 0, fmt.Errorf("Missing field %v", field) 1481 } 1482 1483 if value, err := enclosure.GetFloatValue(); err == nil { 1484 return value, nil 1485 } 1486 1487 // Check if the string value is a number. 1488 if enclosure.Dtype == utils.SS_DT_STRING { 1489 if value, err := strconv.ParseFloat(enclosure.CVal.(string), 64); err == nil { 1490 return value, nil 1491 } 1492 } 1493 1494 return 0, fmt.Errorf("Cannot convert CValueEnclosure %v to float", enclosure) 1495 } 1496 1497 func (self *SortValue) Compare(other *SortValue) (int, error) { 1498 switch self.InterpretAs { 1499 case "ip": 1500 selfIP := net.ParseIP(self.Val) 1501 otherIP := net.ParseIP(other.Val) 1502 if selfIP == nil || otherIP == nil { 1503 return 0, fmt.Errorf("SortValue.Compare: cannot parse IP address") 1504 } 1505 return bytes.Compare(selfIP, otherIP), nil 1506 case "num": 1507 selfFloat, selfErr := strconv.ParseFloat(self.Val, 64) 1508 otherFloat, otherErr := strconv.ParseFloat(other.Val, 64) 1509 if selfErr != nil || otherErr != nil { 1510 return 0, fmt.Errorf("SortValue.Compare: cannot parse %v and %v as float", self.Val, other.Val) 1511 } 1512 1513 if selfFloat == otherFloat { 1514 return 0, nil 1515 } else if selfFloat < otherFloat { 1516 return -1, nil 1517 } else { 1518 return 1, nil 1519 } 1520 case "str": 1521 return strings.Compare(self.Val, other.Val), nil 1522 case "auto", "": 1523 selfFloat, selfErr := strconv.ParseFloat(self.Val, 64) 1524 otherFloat, otherErr := strconv.ParseFloat(other.Val, 64) 1525 if selfErr == nil && otherErr == nil { 1526 if selfFloat == otherFloat { 1527 return 0, nil 1528 } else if selfFloat < otherFloat { 1529 return -1, nil 1530 } else { 1531 return 1, nil 1532 } 1533 } 1534 1535 selfIp := net.ParseIP(self.Val) 1536 otherIp := net.ParseIP(other.Val) 1537 if selfIp != nil && otherIp != nil { 1538 return bytes.Compare(selfIp, otherIp), nil 1539 } 1540 1541 return strings.Compare(self.Val, other.Val), nil 1542 default: 1543 return 0, fmt.Errorf("SortValue.Compare: invalid InterpretAs value: %v", self.InterpretAs) 1544 } 1545 } 1546 1547 // The `ascending` slice should have the same length as `a` and `b`. Moreover, 1548 // each element of `ascending` should be either +1 or -1; +1 means higher 1549 // values get sorted higher, and -1 means lower values get sorted higher. 1550 func CompareSortValueSlices(a []SortValue, b []SortValue, ascending []int) (int, error) { 1551 if len(a) != len(b) || len(a) != len(ascending) { 1552 return 0, fmt.Errorf("CompareSortValueSlices: slices have different lengths") 1553 } 1554 1555 for i := 0; i < len(a); i++ { 1556 comp, err := a[i].Compare(&b[i]) 1557 if err != nil { 1558 return 0, fmt.Errorf("CompareSortValueSlices: %v", err) 1559 } 1560 1561 if comp != 0 { 1562 return comp * ascending[i], nil 1563 } 1564 } 1565 1566 return 0, nil 1567 }