github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/ast/logql/logql.peg (about) 1 { 2 package logql 3 import ( 4 "strconv" 5 "strings" 6 structs "github.com/siglens/siglens/pkg/segment/structs" 7 "github.com/siglens/siglens/pkg/ast" 8 "github.com/siglens/siglens/pkg/segment/utils" 9 "fmt" 10 ) 11 12 func getParseError(err error) error { 13 switch ev := err.(type) { 14 case errList: 15 if pe, ok := ev[0].(*parserError); ok { 16 return &ast.ParseError{ 17 Inner: pe.Inner, 18 Line: pe.pos.line, 19 Column: pe.pos.col, 20 Offset: pe.pos.offset, 21 Prefix: pe.prefix, 22 Expected: pe.expected, 23 } 24 } 25 } 26 return err 27 } 28 29 } 30 31 // LogQL is a Log Query Language parser 32 33 // Grammar rules and functions for LogQL 34 35 Start <- query:Stream? jf:JSONFilter? fs:FilterStream? EOF { 36 var q ast.QueryStruct 37 if jf != nil { 38 q.PipeCommands = jf.(*structs.QueryAggregators) 39 q.PipeCommands.PipeCommandType = structs.OutputTransformType 40 } 41 if query != nil && fs != nil{ 42 finalNode := &ast.Node { 43 NodeType: ast.NodeAnd, 44 Left: query.(*ast.Node), 45 Right: fs.(*ast.Node), 46 } 47 q.SearchFilter = finalNode 48 return q, nil 49 50 } 51 if fs != nil && query == nil { 52 q.SearchFilter = fs.(*ast.Node) 53 return q, nil 54 } 55 56 if query != nil && fs == nil { 57 q.SearchFilter = query.(*ast.Node) 58 return q, nil 59 } 60 61 return q, nil 62 } / COUNT_OVER_TIME "(" query:Stream duration:Duration ")" { 63 var q ast.QueryStruct 64 q.SearchFilter = query.(*ast.Node) 65 aggs := &structs.QueryAggregators{} 66 countCols := []*structs.MeasureAggregator{{MeasureCol: "*", MeasureFunc: utils.Count}} 67 aggs.GroupByRequest = &structs.GroupByRequest{MeasureOperations: countCols, GroupByColumns: []string{"*"}} 68 timeNow := utils.GetCurrentTimeMillis() 69 aggs.TimeHistogram = &structs.TimeBucket{StartTime: timeNow - duration.(uint64), EndTime: timeNow} 70 q.PipeCommands = aggs 71 q.PipeCommands.PipeCommandType = structs.GroupByType 72 return q, nil 73 } 74 75 Stream <- Delimiter q1:Query rest:(Query)* Delimiter space?{ 76 startNode, ok := q1.(*ast.Node) 77 if !ok { 78 return nil, nil 79 } 80 if len(rest.([]interface{})) == 0 { 81 return startNode, nil 82 } 83 finalNode := &ast.Node { 84 NodeType: ast.NodeAnd, 85 Left: startNode, 86 Right: &ast.Node{}, 87 } 88 ptr := finalNode 89 for index, query := range rest.([]interface{}) { 90 if index == len(rest.([]interface{}))-1 { 91 ptr.Right = query.(*ast.Node) 92 } else { 93 ptr.Right = &ast.Node { 94 NodeType: ast.NodeAnd, 95 Left: query.(*ast.Node), 96 Right: &ast.Node{}, 97 } 98 ptr = ptr.Right 99 } 100 } 101 return finalNode, nil 102 } 103 104 FilterStream <- rest:(LogFilter / LabelFilter)* space?{ 105 if rest == nil || len(rest.([]interface{})) == 0 { 106 return nil, nil 107 } 108 list := rest.([]interface{}) 109 if len(list) == 1 { 110 return list[0].(*ast.Node), nil 111 } 112 finalNode := &ast.Node { 113 NodeType: ast.NodeAnd, 114 Left: list[0].(*ast.Node), 115 Right: &ast.Node{}, 116 } 117 ptr := finalNode 118 list = list[1:] 119 for index, filter := range list { 120 if index == len(list)-1 { 121 ptr.Right = filter.(*ast.Node) 122 } else { 123 ptr.Right = &ast.Node { 124 NodeType: ast.NodeAnd, 125 Left: filter.(*ast.Node), 126 Right: &ast.Node{}, 127 } 128 ptr = ptr.Right 129 } 130 } 131 return finalNode, nil 132 } 133 134 Query <- field:Field space? "=" space? field1:Field ','? space? { 135 return &ast.Node{ 136 NodeType: ast.NodeTerminal, 137 Comparison:ast.Comparison{ 138 Op: "=", 139 Field: field.(string), 140 Values: field1, 141 }, 142 }, nil 143 } 144 145 SingleField <- ','? space? field:Field ','?{ 146 return &ast.Node{ 147 NodeType: ast.NodeTerminal, 148 Comparison:ast.Comparison{ 149 Field: field.(string), 150 Values: field, 151 }, 152 }, nil 153 } 154 155 LogFilter <- space? grep:GrepFilter space? field:Field space? { 156 return &ast.Node{ 157 NodeType: ast.NodeTerminal, 158 Comparison:ast.Comparison{ 159 Op: grep.(string), 160 Values: ast.GrepValue{Field: field.(string)}, 161 }, 162 }, nil 163 164 } 165 166 LabelFilter <- space? '|' space? field:Field space? op:opCOMP space? field1:Field ','? space?{ 167 return &ast.Node{ 168 NodeType: ast.NodeTerminal, 169 Comparison:ast.Comparison{ 170 Op: op.(string), 171 Field: field.(string), 172 Values: field1, 173 }, 174 }, nil 175 } 176 177 JSONFilter <- space? '|' space "json" space? rest:(Query)* { 178 if len(rest.([]interface{})) == 0 { 179 return nil, nil 180 } 181 rawIncludeValues := make([]*structs.IncludeValue, 0) 182 mapLabels := make(map[string]string, 0) 183 aggNode := &structs.QueryAggregators{OutputTransforms: &structs.OutputTransforms{}} 184 columsArray := make([]string, 0) 185 for _, query := range rest.([]interface{}) { 186 label := query.(*ast.Node).Comparison.Field 187 expression := strings.Trim(query.(*ast.Node).Comparison.Values.(string), "\"") 188 r := regexp.MustCompile(`^(\w+)\[(\d+)\]$`) 189 isIndexed := r.MatchString(expression) 190 if isIndexed { 191 match := r.FindStringSubmatch(expression) 192 cname := match[1] 193 index, err := strconv.Atoi(match[2]) 194 if err != nil { 195 return nil, fmt.Errorf("ParseJSONRequest: error parsing json array expression") 196 } 197 rawIncludeValues = append(rawIncludeValues, &structs.IncludeValue{Index: index, ColName: cname, Label: label}) 198 columsArray = append(columsArray, cname) 199 } else { 200 mapLabels[expression] = label 201 columsArray = append(columsArray, expression) 202 } 203 } 204 aggNode.OutputTransforms = &structs.OutputTransforms{OutputColumns: &structs.ColumnsRequest{IncludeColumns: columsArray}} 205 aggNode.OutputTransforms.OutputColumns.RenameColumns = mapLabels 206 aggNode.OutputTransforms.OutputColumns.IncludeValues = rawIncludeValues 207 return aggNode, nil 208 } / space? '|' space "logfmt" space? rest:(Query / SingleField)* { 209 aggNode := &structs.QueryAggregators{OutputTransforms: &structs.OutputTransforms{OutputColumns: &structs.ColumnsRequest{}}} 210 rawIncludeValues := make([]*structs.IncludeValue, 0) 211 for _, query := range rest.([]interface{}) { 212 expression := strings.Trim(query.(*ast.Node).Comparison.Values.(string), "\"") 213 newValue := &structs.IncludeValue{ColName: expression, Label: query.(*ast.Node).Comparison.Field} 214 rawIncludeValues = append(rawIncludeValues, newValue) 215 } 216 aggNode.OutputTransforms.OutputColumns.IncludeValues = rawIncludeValues 217 aggNode.OutputTransforms.OutputColumns.Logfmt = true 218 return aggNode, nil 219 } 220 221 Duration <- "[" val:Integer timeUnit:TIME_UNIT "]" { 222 switch rawVal := val.(type) { 223 case json.Number: 224 interval, err := rawVal.Int64() 225 if err != nil { 226 return nil, fmt.Errorf("input is not an integer number") 227 } 228 return uint64(interval) * timeUnit.(uint64), nil 229 default: 230 return nil, fmt.Errorf("input is not a JSON number") 231 } 232 } 233 234 TIME_UNIT <- "ms" { 235 return uint64(1), nil 236 } / "s" { 237 return uint64(1000), nil 238 } / "m" { 239 return uint64(60000), nil 240 } / "h" { 241 return uint64(60 * 60000), nil 242 } / "d" { 243 return uint64(24 * 60 * 60000), nil 244 } / "w" { 245 return uint64(7 * 24 * 60 * 60000), nil 246 } / "y" { 247 return uint64(365 * 24 * 60 * 60000), nil 248 } 249 250 opCOMP <- opCustom 251 / "==" { 252 return "=", nil 253 } / "=~" { 254 return "=", nil 255 } / "<=" { 256 return string(c.text), nil 257 } / ">=" { 258 return string(c.text), nil 259 } / "=" { 260 return string(c.text), nil 261 } / "<" { 262 return string(c.text), nil 263 } / ">" { 264 return string(c.text), nil 265 } / "!=" { 266 return string(c.text), nil 267 } / "!~" { 268 return "!=", nil 269 } 270 271 GrepFilter <- "|=" { 272 return "=", nil 273 } / "!=" { 274 return string(c.text), nil 275 } / "|~" { 276 return "=", nil 277 } / "!~" { 278 return "!=", nil 279 } 280 281 282 opCustom <- '=' opname:[a-z]i+ '='{ 283 return ast.OpNameToString(c.text) 284 } 285 286 287 LetOpr <- [>] '=' { 288 return utils.LetGreaterThanOrEqualTo, nil 289 } / '>' { 290 return utils.LetGreaterThan, nil 291 } / [<] '=' { 292 return utils.LetLessThanOrEqualTo, nil 293 } / '<' { 294 return utils.LetLessThan, nil 295 } / [=] '=' { 296 return utils.LetEquals, nil 297 } / [!] '=' { 298 return utils.LetNotEquals, nil 299 } / '+' { 300 return utils.LetAdd, nil 301 } / '-' { 302 return utils.LetSubtract, nil 303 } / '/' { 304 return utils.LetDivide, nil 305 } / '*' { 306 return utils.LetMultiply, nil 307 } / '%' { 308 return utils.LetModulo, nil 309 } 310 311 LetIdentifier <- '-'? Float { 312 var dte utils.DtypeEnclosure 313 dte.Dtype = utils.SS_DT_FLOAT 314 dte.FloatVal,_ = strconv.ParseFloat(string(c.text), 64) 315 dte.StringVal = string(c.text) 316 return &dte, nil 317 } / '-'+ Integer { 318 var dte utils.DtypeEnclosure 319 dte.Dtype = utils.SS_DT_SIGNED_NUM 320 dte.SignedVal,_ = strconv.ParseInt(string(c.text),10, 64) 321 dte.StringVal = string(c.text) 322 return &dte, nil 323 } / Integer { 324 var dte utils.DtypeEnclosure 325 dte.Dtype = utils.SS_DT_UNSIGNED_NUM 326 dte.UnsignedVal,_ = strconv.ParseUint(string(c.text),10, 64) 327 dte.StringVal = string(c.text) 328 return &dte, nil 329 } / QuotedValue { 330 var dte utils.DtypeEnclosure 331 dte.Dtype = utils.SS_DT_STRING 332 dte.BoolVal = 1 333 dte.StringVal = string(c.text) 334 return &dte, nil 335 } 336 337 BoolValue <- "false" / "true" 338 339 COUNT_OVER_TIME <- "count_over_time" 340 341 Field <- Value / pieces:FieldPiece { 342 if pieces == nil { 343 return "*", nil 344 } 345 346 return string(c.text), nil 347 } 348 349 Identifier <- 350 [a-zA-Z0-9_@./*]i+ { 351 return string(c.text), nil 352 } 353 354 Value <- val:( 355 Float 356 / Integer 357 358 ){ 359 return ast.MakeValue(val) 360 } 361 362 Integer <- [+-]? [0-9]+ { 363 return json.Number(string(c.text)), nil 364 } 365 366 Float <- [+-]? ([0-9]* "." [0-9]+ ) { 367 return json.Number(string(c.text)), nil 368 } 369 370 FieldPiece <- QuotedFieldPiece / UnquotedFieldPiece / Star 371 372 UnquotedFieldPiece <- [-a-zA-Z0-9$&,?#%_@;[\]{}+-./*:]i+ { 373 return string(c.text), nil 374 } 375 376 QuotedFieldPiece <- QuotedValue 377 378 Star <- '*' { 379 return "*", nil 380 } 381 QuotedValue <- '"' ( !EscapedChar . / '\\' EscapeSequence )* '"' { 382 c.text = bytes.Replace(c.text, []byte(`\/`), []byte(`/`), -1) 383 return strconv.Unquote(string(c.text)) 384 } 385 EscapedChar <- [\x00-\x1f"\\] 386 387 EscapeSequence <- SingleCharEscape / UnicodeEscape 388 389 SingleCharEscape <- ["\\/bfnrt] 390 391 UnicodeEscape <- 'u' HexDigit HexDigit HexDigit HexDigit 392 393 HexDigit <- [0-9a-f]i 394 395 /********************** 396 WHITESPACE AND TERMINAL 397 ***********************/ 398 EOF = !. 399 400 _ "whitespace" <- [ \n\t\r]* 401 402 space <- [ \n\t\r]+ 403 404 Delimiter <- "{" / "}" 405