github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/ast/logql/logql.peg (about)

     1  {
     2  package logql
     3  import (
     4  	"strconv"
     5  	"strings"
     6  	structs "github.com/siglens/siglens/pkg/segment/structs"
     7  	"github.com/siglens/siglens/pkg/ast"
     8  	"github.com/siglens/siglens/pkg/segment/utils"
     9      "fmt"
    10  )
    11  
    12  func getParseError(err error) error {
    13  	switch ev := err.(type) {
    14  	case errList:
    15  		if pe, ok := ev[0].(*parserError); ok {
    16  			return &ast.ParseError{
    17  				Inner:    pe.Inner,
    18  				Line:     pe.pos.line,
    19  				Column:   pe.pos.col,
    20  				Offset:   pe.pos.offset,
    21  				Prefix:   pe.prefix,
    22  				Expected: pe.expected,
    23  			}
    24  		}
    25  	}
    26  	return err
    27  }
    28  
    29  }
    30  
    31  // LogQL is a Log Query Language parser
    32  
    33  // Grammar rules and functions for LogQL
    34  
    35  Start <- query:Stream? jf:JSONFilter? fs:FilterStream? EOF {
    36      var q ast.QueryStruct
    37      if jf != nil {
    38          q.PipeCommands = jf.(*structs.QueryAggregators)
    39          q.PipeCommands.PipeCommandType = structs.OutputTransformType
    40      }
    41      if query != nil && fs != nil{
    42          finalNode := &ast.Node {
    43          NodeType: ast.NodeAnd,
    44          Left: query.(*ast.Node),
    45          Right: fs.(*ast.Node),
    46          }
    47          q.SearchFilter = finalNode
    48          return q, nil
    49  
    50      }
    51      if fs != nil && query == nil {
    52          q.SearchFilter = fs.(*ast.Node)
    53          return q, nil
    54      }
    55  
    56      if query != nil && fs == nil {
    57          q.SearchFilter = query.(*ast.Node)
    58          return q, nil
    59      }
    60  
    61      return q, nil 
    62  } / COUNT_OVER_TIME "(" query:Stream duration:Duration ")" {
    63      var q ast.QueryStruct
    64      q.SearchFilter = query.(*ast.Node)
    65      aggs := &structs.QueryAggregators{}
    66  	countCols := []*structs.MeasureAggregator{{MeasureCol: "*", MeasureFunc: utils.Count}}
    67  	aggs.GroupByRequest = &structs.GroupByRequest{MeasureOperations: countCols, GroupByColumns: []string{"*"}}
    68  	timeNow := utils.GetCurrentTimeMillis()
    69  	aggs.TimeHistogram = &structs.TimeBucket{StartTime: timeNow - duration.(uint64), EndTime: timeNow}
    70      q.PipeCommands = aggs
    71      q.PipeCommands.PipeCommandType = structs.GroupByType
    72      return q, nil
    73  }
    74  
    75  Stream <- Delimiter q1:Query rest:(Query)* Delimiter space?{
    76      startNode, ok := q1.(*ast.Node)
    77      if !ok {
    78          return nil, nil
    79      }
    80      if len(rest.([]interface{})) == 0 {
    81          return startNode, nil
    82      }
    83      finalNode := &ast.Node {
    84          NodeType: ast.NodeAnd,
    85          Left: startNode,
    86          Right: &ast.Node{},
    87      }
    88      ptr := finalNode
    89      for index, query := range rest.([]interface{}) {
    90          if index == len(rest.([]interface{}))-1 {
    91              ptr.Right = query.(*ast.Node)
    92          } else {
    93              ptr.Right = &ast.Node {
    94              NodeType: ast.NodeAnd,
    95              Left: query.(*ast.Node),
    96              Right: &ast.Node{},
    97              }
    98              ptr = ptr.Right
    99          }
   100      }
   101      return finalNode, nil
   102  }
   103  
   104  FilterStream <- rest:(LogFilter / LabelFilter)* space?{
   105      if rest == nil || len(rest.([]interface{})) == 0 {
   106          return nil, nil
   107      }
   108      list := rest.([]interface{})
   109      if len(list) == 1 {
   110          return list[0].(*ast.Node), nil 
   111      }
   112      finalNode := &ast.Node {
   113          NodeType: ast.NodeAnd,
   114          Left: list[0].(*ast.Node),
   115          Right: &ast.Node{},
   116      }
   117      ptr := finalNode
   118      list = list[1:]
   119      for index, filter := range list {
   120          if index == len(list)-1 {
   121              ptr.Right = filter.(*ast.Node)
   122          } else {
   123              ptr.Right = &ast.Node {
   124              NodeType: ast.NodeAnd,
   125              Left: filter.(*ast.Node),
   126              Right: &ast.Node{},
   127              }
   128              ptr = ptr.Right
   129          }
   130      }
   131      return finalNode, nil
   132  }
   133  
   134  Query <-  field:Field space? "=" space? field1:Field ','? space? {
   135      return &ast.Node{
   136          NodeType: ast.NodeTerminal,
   137          Comparison:ast.Comparison{
   138              Op: "=",
   139              Field: field.(string),
   140              Values: field1,
   141          },
   142      }, nil
   143  }
   144  
   145  SingleField <-  ','? space? field:Field ','?{
   146      return &ast.Node{
   147          NodeType: ast.NodeTerminal,
   148          Comparison:ast.Comparison{
   149              Field: field.(string), 
   150              Values: field,
   151          },
   152      }, nil
   153  }
   154  
   155  LogFilter <- space? grep:GrepFilter space? field:Field space? {
   156      return &ast.Node{
   157          NodeType: ast.NodeTerminal,
   158          Comparison:ast.Comparison{
   159              Op: grep.(string),
   160              Values: ast.GrepValue{Field: field.(string)},
   161          },
   162      }, nil
   163  
   164  }
   165  
   166  LabelFilter <- space? '|' space? field:Field space? op:opCOMP space? field1:Field ','? space?{
   167      return &ast.Node{
   168          NodeType: ast.NodeTerminal,
   169          Comparison:ast.Comparison{
   170              Op: op.(string),
   171              Field: field.(string),
   172              Values: field1,
   173          },
   174      }, nil
   175  }
   176  
   177  JSONFilter <- space? '|' space "json" space? rest:(Query)* {
   178      if len(rest.([]interface{})) == 0 {
   179          return nil, nil
   180      }
   181      rawIncludeValues := make([]*structs.IncludeValue, 0)
   182      mapLabels := make(map[string]string, 0)
   183      aggNode := &structs.QueryAggregators{OutputTransforms: &structs.OutputTransforms{}}
   184      columsArray := make([]string, 0)
   185  	for _, query := range rest.([]interface{}) {
   186          label := query.(*ast.Node).Comparison.Field
   187  		expression := strings.Trim(query.(*ast.Node).Comparison.Values.(string), "\"")
   188  		r := regexp.MustCompile(`^(\w+)\[(\d+)\]$`)
   189  		isIndexed := r.MatchString(expression)
   190  		if isIndexed {
   191  			match := r.FindStringSubmatch(expression)
   192  			cname := match[1]
   193  			index, err := strconv.Atoi(match[2])
   194  			if err != nil {
   195  				return nil, fmt.Errorf("ParseJSONRequest: error parsing json array expression")
   196  			}
   197              rawIncludeValues = append(rawIncludeValues, &structs.IncludeValue{Index: index, ColName: cname, Label: label})
   198              columsArray = append(columsArray, cname)
   199  		} else {
   200              mapLabels[expression] = label
   201              columsArray = append(columsArray, expression)
   202  		}
   203  	}
   204      aggNode.OutputTransforms = &structs.OutputTransforms{OutputColumns: &structs.ColumnsRequest{IncludeColumns: columsArray}}
   205  	aggNode.OutputTransforms.OutputColumns.RenameColumns = mapLabels
   206      aggNode.OutputTransforms.OutputColumns.IncludeValues = rawIncludeValues
   207  	return aggNode, nil
   208  } / space? '|' space "logfmt" space? rest:(Query / SingleField)* {
   209      aggNode := &structs.QueryAggregators{OutputTransforms: &structs.OutputTransforms{OutputColumns: &structs.ColumnsRequest{}}}
   210      rawIncludeValues := make([]*structs.IncludeValue, 0)
   211  	for _, query := range rest.([]interface{}) {
   212          expression := strings.Trim(query.(*ast.Node).Comparison.Values.(string), "\"")
   213          newValue := &structs.IncludeValue{ColName: expression, Label: query.(*ast.Node).Comparison.Field}
   214          rawIncludeValues = append(rawIncludeValues, newValue)
   215      }
   216  	aggNode.OutputTransforms.OutputColumns.IncludeValues = rawIncludeValues
   217      aggNode.OutputTransforms.OutputColumns.Logfmt = true
   218  	return aggNode, nil
   219  }
   220  
   221  Duration <- "[" val:Integer timeUnit:TIME_UNIT "]" {
   222      switch rawVal := val.(type) {
   223  	case json.Number:
   224  		interval, err := rawVal.Int64()
   225          if err != nil {
   226              return nil, fmt.Errorf("input is not an integer number")
   227          }
   228          return uint64(interval) * timeUnit.(uint64), nil 
   229  	default:
   230  		return nil, fmt.Errorf("input is not a JSON number")
   231  	}
   232  }
   233  
   234  TIME_UNIT <- "ms" {
   235      return uint64(1), nil
   236  } / "s" {
   237      return uint64(1000), nil
   238  } / "m" {
   239      return uint64(60000), nil
   240  } / "h" {
   241      return uint64(60 * 60000), nil
   242  } / "d" {
   243      return uint64(24 * 60 * 60000), nil
   244  } / "w" {
   245      return uint64(7 * 24 * 60 * 60000), nil
   246  } / "y" {
   247      return uint64(365 * 24 * 60 * 60000), nil
   248  }
   249  
   250  opCOMP <- opCustom
   251  / "==" {
   252      return "=", nil
   253  } / "=~" {
   254      return "=", nil
   255  } / "<=" {
   256      return string(c.text), nil
   257  } / ">=" {
   258      return string(c.text), nil
   259  } / "=" {
   260      return string(c.text), nil
   261  } / "<" {
   262      return string(c.text), nil
   263  } / ">" {
   264      return string(c.text), nil
   265  } / "!=" {
   266      return string(c.text), nil
   267  } / "!~" {
   268      return "!=", nil 
   269  }
   270  
   271  GrepFilter <- "|=" {
   272      return "=", nil
   273  } / "!=" {
   274      return string(c.text), nil
   275  } / "|~" {
   276      return "=", nil 
   277  } / "!~" {
   278      return "!=", nil 
   279  }
   280  
   281  
   282  opCustom <- '=' opname:[a-z]i+ '='{
   283      return ast.OpNameToString(c.text)
   284  }
   285  
   286  
   287  LetOpr <- [>] '=' {
   288      return utils.LetGreaterThanOrEqualTo, nil
   289  } / '>' {
   290      return utils.LetGreaterThan, nil
   291  } / [<] '=' {
   292      return utils.LetLessThanOrEqualTo, nil
   293  } / '<' {
   294      return utils.LetLessThan, nil
   295  } / [=] '=' {
   296      return utils.LetEquals, nil
   297  } / [!] '=' {
   298      return utils.LetNotEquals, nil
   299  } / '+' {
   300      return utils.LetAdd, nil
   301  } / '-' {
   302      return utils.LetSubtract, nil
   303  } / '/' {
   304      return utils.LetDivide, nil
   305  } / '*' {
   306      return utils.LetMultiply, nil
   307  } / '%' {
   308      return utils.LetModulo, nil
   309  }
   310  
   311  LetIdentifier <- '-'? Float {
   312      var dte utils.DtypeEnclosure
   313      dte.Dtype = utils.SS_DT_FLOAT
   314      dte.FloatVal,_ = strconv.ParseFloat(string(c.text), 64)
   315      dte.StringVal = string(c.text)
   316      return &dte, nil
   317  } / '-'+ Integer {
   318      var dte utils.DtypeEnclosure
   319      dte.Dtype = utils.SS_DT_SIGNED_NUM
   320      dte.SignedVal,_ = strconv.ParseInt(string(c.text),10, 64)
   321      dte.StringVal = string(c.text)
   322      return &dte, nil
   323  } /  Integer {
   324      var dte utils.DtypeEnclosure
   325      dte.Dtype = utils.SS_DT_UNSIGNED_NUM
   326      dte.UnsignedVal,_ = strconv.ParseUint(string(c.text),10, 64)
   327      dte.StringVal = string(c.text)
   328      return &dte, nil
   329  } / QuotedValue {
   330      var dte utils.DtypeEnclosure
   331      dte.Dtype = utils.SS_DT_STRING
   332      dte.BoolVal = 1
   333      dte.StringVal = string(c.text)
   334      return &dte, nil
   335  }
   336  
   337  BoolValue <- "false" / "true"
   338  
   339  COUNT_OVER_TIME <- "count_over_time" 
   340  
   341  Field <- Value / pieces:FieldPiece {
   342      if pieces == nil {
   343  		return "*", nil
   344  	}
   345  
   346      return string(c.text), nil
   347  }
   348  
   349  Identifier <-
   350     [a-zA-Z0-9_@./*]i+ { 
   351        return string(c.text), nil
   352  }
   353  
   354  Value <- val:(
   355      Float
   356      / Integer
   357      
   358      ){
   359      return ast.MakeValue(val)
   360    }
   361  
   362  Integer <- [+-]? [0-9]+ {
   363      return json.Number(string(c.text)), nil
   364    }
   365  
   366  Float <- [+-]? ([0-9]* "." [0-9]+ ) {
   367        return json.Number(string(c.text)), nil
   368      }
   369  
   370  FieldPiece <- QuotedFieldPiece / UnquotedFieldPiece / Star
   371  
   372  UnquotedFieldPiece <- [-a-zA-Z0-9$&,?#%_@;[\]{}+-./*:]i+ {
   373      return string(c.text), nil
   374  }
   375  
   376  QuotedFieldPiece <- QuotedValue
   377  
   378  Star <- '*' {
   379      return "*", nil
   380  }
   381  QuotedValue <- '"' ( !EscapedChar . / '\\' EscapeSequence )* '"' {
   382      c.text = bytes.Replace(c.text, []byte(`\/`), []byte(`/`), -1)
   383      return strconv.Unquote(string(c.text))
   384  }
   385  EscapedChar <- [\x00-\x1f"\\]
   386  
   387  EscapeSequence <- SingleCharEscape / UnicodeEscape
   388  
   389  SingleCharEscape <- ["\\/bfnrt]
   390  
   391  UnicodeEscape <- 'u' HexDigit HexDigit HexDigit HexDigit
   392  
   393  HexDigit <- [0-9a-f]i
   394  
   395  /**********************
   396  WHITESPACE AND TERMINAL
   397  ***********************/
   398  EOF = !.
   399  
   400  _ "whitespace" <- [ \n\t\r]*
   401  
   402  space <- [ \n\t\r]+
   403  
   404  Delimiter <- "{" / "}"
   405