github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/pkg/ast/logql/tests/logqlParse_test.go (about)

     1  /*
     2  Copyright 2023.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package tests
    18  
    19  import (
    20  	"testing"
    21  
    22  	"github.com/siglens/siglens/pkg/ast"
    23  	"github.com/siglens/siglens/pkg/ast/logql"
    24  	"github.com/siglens/siglens/pkg/ast/pipesearch"
    25  	"github.com/siglens/siglens/pkg/segment/structs"
    26  	"github.com/stretchr/testify/assert"
    27  )
    28  
    29  func Test_ParseStream(t *testing.T) {
    30  	astNode := &structs.ASTNode{}
    31  	json_body := []byte(`{something="another"}`)
    32  	res, err := logql.Parse("", json_body)
    33  	queryJson := res.(ast.QueryStruct).SearchFilter
    34  	assert.Nil(t, err)
    35  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
    36  	assert.Nil(t, err)
    37  	assert.NotNil(t, queryJson)
    38  	assert.Equal(t, queryJson.Comparison.Field, "something")
    39  	assert.Equal(t, queryJson.Comparison.Values, "\"another\"")
    40  	assert.Equal(t, queryJson.Comparison.Op, "=")
    41  
    42  	json_body = []byte(`{something="another", another="thing"}`)
    43  	res, err = logql.Parse("", json_body)
    44  	queryJson = res.(ast.QueryStruct).SearchFilter
    45  	assert.Nil(t, err)
    46  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
    47  	assert.Nil(t, err)
    48  	assert.NotNil(t, queryJson)
    49  	assert.Equal(t, queryJson.Left.Comparison.Field, "something")
    50  	assert.Equal(t, queryJson.Left.Comparison.Values, "\"another\"")
    51  	assert.Equal(t, queryJson.Right.Comparison.Field, "another")
    52  	assert.Equal(t, queryJson.Right.Comparison.Values, "\"thing\"")
    53  	assert.Equal(t, queryJson.NodeType, ast.NodeAnd)
    54  }
    55  
    56  func Test_ParseLabelFilter(t *testing.T) {
    57  	astNode := &structs.ASTNode{}
    58  	json_body := []byte(`{something="another"} | another >= thing`)
    59  	res, err := logql.Parse("", json_body)
    60  	queryJson := res.(ast.QueryStruct).SearchFilter
    61  	assert.Nil(t, err)
    62  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
    63  	assert.Nil(t, err)
    64  	assert.Equal(t, queryJson.Left.Comparison.Field, "something")
    65  	assert.Equal(t, queryJson.Left.Comparison.Values, "\"another\"")
    66  	assert.Equal(t, queryJson.Right.Comparison.Field, "another")
    67  	assert.Equal(t, queryJson.Right.Comparison.Values, "thing")
    68  }
    69  
    70  func Test_ParseLogFilter(t *testing.T) {
    71  	astNode := &structs.ASTNode{}
    72  	json_body := []byte(`{gender="female",city="Fresno"} != "batch-212"`)
    73  	res, err := logql.Parse("", json_body)
    74  	queryJson := res.(ast.QueryStruct).SearchFilter
    75  	assert.Nil(t, err)
    76  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
    77  	assert.Nil(t, err)
    78  	assert.Equal(t, queryJson.Right.Comparison.Values, ast.GrepValue{Field: "\"batch-212\""})
    79  	assert.Equal(t, queryJson.Left.Right.Comparison.Values, "\"Fresno\"")
    80  	assert.Equal(t, queryJson.Left.Left.Comparison.Values, "\"female\"")
    81  }
    82  
    83  func Test_ParseLogAndLabelFilter(t *testing.T) {
    84  	astNode := &structs.ASTNode{}
    85  	json_body := []byte(`{gender="female",city="Fresno"} |= "batch-212" | another >= thing`)
    86  	res, err := logql.Parse("", json_body)
    87  	assert.Nil(t, err)
    88  	queryJson := res.(ast.QueryStruct).SearchFilter
    89  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
    90  	assert.Nil(t, err)
    91  	assert.Equal(t, queryJson.Right.Right.Comparison.Values, "thing")
    92  	assert.Equal(t, queryJson.Right.Left.Comparison.Values, ast.GrepValue{Field: "\"batch-212\""})
    93  	assert.Equal(t, queryJson.Left.Right.Comparison.Values, "\"Fresno\"")
    94  	assert.Equal(t, queryJson.Left.Left.Comparison.Values, "\"female\"")
    95  }
    96  
    97  func Test_ParseLogfmtKeyword(t *testing.T) {
    98  	astNode := &structs.ASTNode{}
    99  	json_body := []byte(`{gender="female",city="Fresno"} | logfmt city_life="city", single_gender="gender", host`)
   100  	res, err := logql.Parse("", json_body)
   101  	assert.Nil(t, err)
   102  	queryJson := res.(ast.QueryStruct).SearchFilter
   103  	pipeCommands := res.(ast.QueryStruct).PipeCommands
   104  	testIncludeValues := append(make([]*structs.IncludeValue, 0), &structs.IncludeValue{ColName: "city", Label: "city_life"}, &structs.IncludeValue{ColName: "gender", Label: "single_gender"}, &structs.IncludeValue{ColName: "host", Label: "host"})
   105  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
   106  	assert.Nil(t, err)
   107  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeValues, testIncludeValues)
   108  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.Logfmt, true)
   109  	assert.Equal(t, queryJson.Left.Comparison.Values, "\"female\"")
   110  	assert.Equal(t, queryJson.Right.Comparison.Values, "\"Fresno\"")
   111  
   112  	json_body = []byte(`{gender="female"} | logfmt `)
   113  	res, err = logql.Parse("", json_body)
   114  	assert.Nil(t, err)
   115  	pipeCommands = res.(ast.QueryStruct).PipeCommands
   116  	queryJson = res.(ast.QueryStruct).SearchFilter
   117  	astNode = &structs.ASTNode{}
   118  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
   119  	assert.Nil(t, err)
   120  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.Logfmt, true)
   121  	assert.Equal(t, len(pipeCommands.OutputTransforms.OutputColumns.IncludeValues), 0)
   122  	assert.Equal(t, queryJson.Comparison.Values, "\"female\"")
   123  }
   124  
   125  func Test_ParseJSONKeyword(t *testing.T) {
   126  	astNode := &structs.ASTNode{}
   127  	json_body := []byte(`{gender="female",city="Fresno"} | json city_life="city", single_gender="gender[0]"`)
   128  	res, err := logql.Parse("", json_body)
   129  	assert.Nil(t, err)
   130  	queryJson := res.(ast.QueryStruct).SearchFilter
   131  	pipeCommands := res.(ast.QueryStruct).PipeCommands
   132  	testOutputColumns := append(make([]string, 0), "city", "gender")
   133  	testRenameColumns := make(map[string]string)
   134  	testRenameColumns["city"] = "city_life"
   135  	testIncludeValues := append(make([]*structs.IncludeValue, 0), &structs.IncludeValue{Index: 0, ColName: "gender", Label: "single_gender"})
   136  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
   137  	assert.Nil(t, err)
   138  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeColumns, testOutputColumns)
   139  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.RenameColumns, testRenameColumns)
   140  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeValues, testIncludeValues)
   141  	assert.Equal(t, queryJson.Left.Comparison.Values, "\"female\"")
   142  	assert.Equal(t, queryJson.Right.Comparison.Values, "\"Fresno\"")
   143  
   144  	json_body = []byte(`{gender="female"} | json `)
   145  	res, err = logql.Parse("", json_body)
   146  	assert.Nil(t, err)
   147  	pipeCommands = res.(ast.QueryStruct).PipeCommands
   148  	queryJson = res.(ast.QueryStruct).SearchFilter
   149  	astNode = &structs.ASTNode{}
   150  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
   151  	assert.Nil(t, err)
   152  	assert.Equal(t, queryJson.Comparison.Values, "\"female\"")
   153  	assert.Nil(t, pipeCommands)
   154  }
   155  
   156  func Test_ParseJSONKeywordAndFilters(t *testing.T) {
   157  	astNode := &structs.ASTNode{}
   158  	json_body := []byte(`{gender="female",city="Fresno"} | json city_life="city", single_gender="gender[0]" |= "batch-212"`)
   159  	res, err := logql.Parse("", json_body)
   160  	assert.Nil(t, err)
   161  	queryJson := res.(ast.QueryStruct).SearchFilter
   162  	pipeCommands := res.(ast.QueryStruct).PipeCommands
   163  	testOutputColumns := append(make([]string, 0), "city", "gender")
   164  	testRenameColumns := make(map[string]string)
   165  	testRenameColumns["city"] = "city_life"
   166  	testIncludeValues := append(make([]*structs.IncludeValue, 0), &structs.IncludeValue{Index: 0, ColName: "gender", Label: "single_gender"})
   167  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
   168  	assert.Nil(t, err)
   169  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeColumns, testOutputColumns)
   170  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.RenameColumns, testRenameColumns)
   171  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeValues, testIncludeValues)
   172  	assert.Equal(t, queryJson.Left.Left.Comparison.Values, "\"female\"")
   173  	assert.Equal(t, queryJson.Left.Right.Comparison.Values, "\"Fresno\"")
   174  	assert.Equal(t, queryJson.Right.Comparison.Values, ast.GrepValue{Field: "\"batch-212\""})
   175  
   176  	json_body = []byte(`{gender="female",city="Fresno"} | json city_life="city", single_gender="gender[0]" |= "batch-212" | another >= thing`)
   177  	res, err = logql.Parse("", json_body)
   178  	assert.Nil(t, err)
   179  	queryJson = res.(ast.QueryStruct).SearchFilter
   180  	pipeCommands = res.(ast.QueryStruct).PipeCommands
   181  	astNode = &structs.ASTNode{}
   182  	err = pipesearch.SearchQueryToASTnode(queryJson, astNode, 0)
   183  	assert.Nil(t, err)
   184  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeColumns, testOutputColumns)
   185  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.RenameColumns, testRenameColumns)
   186  	assert.Equal(t, pipeCommands.OutputTransforms.OutputColumns.IncludeValues, testIncludeValues)
   187  	assert.Equal(t, queryJson.Left.Left.Comparison.Values, "\"female\"")
   188  	assert.Equal(t, queryJson.Left.Right.Comparison.Values, "\"Fresno\"")
   189  	assert.Equal(t, queryJson.Right.Left.Comparison.Values, ast.GrepValue{Field: "\"batch-212\""})
   190  	assert.Equal(t, queryJson.Right.Right.Comparison.Values, "thing")
   191  
   192  }