github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/logql/syntax/lex_test.go (about)

     1  package syntax
     2  
     3  import (
     4  	"strings"
     5  	"testing"
     6  	"text/scanner"
     7  	"time"
     8  
     9  	"github.com/stretchr/testify/require"
    10  )
    11  
    12  func TestLex(t *testing.T) {
    13  	for _, tc := range []struct {
    14  		input    string
    15  		expected []int
    16  	}{
    17  		{`{foo="bar"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
    18  		{"{foo=\"bar\"} |~  `\\w+`", []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING}},
    19  		{`{foo="bar"} |~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING}},
    20  		{`{foo="bar"} |~ "\\w+" | latency > 250ms`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, DURATION}},
    21  		{`{foo="bar"} |~ "\\w+" | foo = 0ms`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, EQ, DURATION}},
    22  		{`{foo="bar"} |~ "\\w+" | latency > 1h15m30.918273645s`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, DURATION}},
    23  		{`{foo="bar"} |~ "\\w+" | latency > 1h0.0m0s`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, DURATION}},
    24  		{
    25  			`{foo="bar"} |~ "\\w+" | latency > 1h0.0m0s or foo == 4.00 and bar ="foo"`,
    26  			[]int{
    27  				OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING,
    28  				PIPE, IDENTIFIER, GT, DURATION, OR, IDENTIFIER, CMP_EQ, NUMBER, AND, IDENTIFIER, EQ, STRING,
    29  			},
    30  		},
    31  		{
    32  			`{foo="bar"} |~ "\\w+" | duration > 1h0.0m0s or avg == 4.00 and bar ="foo"`,
    33  			[]int{
    34  				OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING,
    35  				PIPE, IDENTIFIER, GT, DURATION, OR, IDENTIFIER, CMP_EQ, NUMBER, AND, IDENTIFIER, EQ, STRING,
    36  			},
    37  		},
    38  		{
    39  			`{foo="bar"} |~ "\\w+" | latency > 1h0.0m0s or foo == 4.00 and bar ="foo" | unwrap foo`,
    40  			[]int{
    41  				OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING,
    42  				PIPE, IDENTIFIER, GT, DURATION, OR, IDENTIFIER, CMP_EQ, NUMBER, AND, IDENTIFIER, EQ, STRING, PIPE, UNWRAP, IDENTIFIER,
    43  			},
    44  		},
    45  		{`{foo="bar"} |~ "\\w+" | size > 250kB`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, BYTES}},
    46  		{
    47  			`{foo="bar"} |~ "\\w+" | size > 250kB and latency <= 1h15m30s or bar=1`,
    48  			[]int{
    49  				OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE,
    50  				IDENTIFIER, GT, BYTES, AND, IDENTIFIER, LTE, DURATION, OR, IDENTIFIER, EQ, NUMBER,
    51  			},
    52  		},
    53  		{
    54  			`{foo="bar"} |~ "\\w+" | size > 200MiB or foo == 4.00`,
    55  			[]int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, BYTES, OR, IDENTIFIER, CMP_EQ, NUMBER},
    56  		},
    57  		{`{ foo = "bar" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
    58  		{`{ foo != "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NEQ, STRING, CLOSE_BRACE}},
    59  		{`{ foo =~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, RE, STRING, CLOSE_BRACE}},
    60  		{`{ foo !~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NRE, STRING, CLOSE_BRACE}},
    61  		{`{ foo = "bar", bar != "baz" }`, []int{
    62  			OPEN_BRACE, IDENTIFIER, EQ, STRING,
    63  			COMMA, IDENTIFIER, NEQ, STRING, CLOSE_BRACE,
    64  		}},
    65  		{`{ foo = "ba\"r" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
    66  		{`rate({foo="bar"}[10s])`, []int{RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS}},
    67  		{`rate_counter({foo="bar"} | unwrap foo[10s])`, []int{RATE_COUNTER, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, UNWRAP, IDENTIFIER, RANGE, CLOSE_PARENTHESIS}},
    68  		{`count_over_time({foo="bar"}[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS}},
    69  		{`count_over_time({foo="bar"} |~ "\\w+" | unwrap foo[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, UNWRAP, IDENTIFIER, RANGE, CLOSE_PARENTHESIS}},
    70  		{`sum(count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}},
    71  		{`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, NUMBER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}},
    72  		{`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, NUMBER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}},
    73  		{`sum(max(rate({foo="bar"}[5m])) by (foo,bar)) by (foo)`, []int{SUM, OPEN_PARENTHESIS, MAX, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, CLOSE_PARENTHESIS}},
    74  		{`{foo="bar"} #|~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
    75  		{`#{foo="bar"} |~ "\\w+"`, []int{}},
    76  		{`{foo="#"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
    77  		{`{foo="bar"}|logfmt|ip="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}},
    78  		{`{foo="bar"}|logfmt|rate="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}},
    79  		{`{foo="bar"}|logfmt|b=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}},
    80  		{`{foo="bar"}|logfmt|=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE_EXACT, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}},
    81  		{`ip`, []int{IDENTIFIER}},
    82  		{`rate`, []int{IDENTIFIER}},
    83  		{`{foo="bar"} | json | baz="#"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, PIPE, IDENTIFIER, EQ, STRING}},
    84  		{`{foo="bar"}
    85  					# |~ "\\w+"
    86  					| json`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON}},
    87  		{`{foo="bar"} | json code="response.code", param="request.params[0]"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, IDENTIFIER, EQ, STRING, COMMA, IDENTIFIER, EQ, STRING}},
    88  	} {
    89  		t.Run(tc.input, func(t *testing.T) {
    90  			actual := []int{}
    91  			l := lexer{
    92  				Scanner: scanner.Scanner{
    93  					Mode: scanner.SkipComments | scanner.ScanStrings,
    94  				},
    95  			}
    96  			l.Init(strings.NewReader(tc.input))
    97  			var lval exprSymType
    98  			for {
    99  				tok := l.Lex(&lval)
   100  				if tok == 0 {
   101  					break
   102  				}
   103  				actual = append(actual, tok)
   104  			}
   105  			require.Equal(t, tc.expected, actual)
   106  		})
   107  	}
   108  }
   109  
   110  func Test_isFunction(t *testing.T) {
   111  	tests := []struct {
   112  		next string
   113  		want bool
   114  	}{
   115  		{"   (", true},
   116  		{"(", true},
   117  		{"by (", true},
   118  		{"by(", true},
   119  		{"by    (", true},
   120  		{"  by (", true},
   121  		{" by(", true},
   122  		{"by    (", true},
   123  		{"without (", true},
   124  		{"without(", true},
   125  		{"without    (", true},
   126  		{"  without (", true},
   127  		{" without(", true},
   128  		{"without    (", true},
   129  		{"   ( whatever is this", true},
   130  		{"   (foo,bar)", true},
   131  		{"\r\n \t\t\r\n \n   (foo,bar)", true},
   132  
   133  		{"  foo (", false},
   134  		{"123", false},
   135  		{"", false},
   136  		{"   ", false},
   137  		{"   )(", false},
   138  		{"byfoo", false},
   139  		{"without foo", false},
   140  	}
   141  	for _, tt := range tests {
   142  		t.Run(tt.next, func(t *testing.T) {
   143  			sc := scanner.Scanner{}
   144  			sc.Init(strings.NewReader(tt.next))
   145  			if got := isFunction(sc); got != tt.want {
   146  				t.Errorf("isFunction() = %v, want %v", got, tt.want)
   147  			}
   148  		})
   149  	}
   150  }
   151  
   152  func Test_parseDuration(t *testing.T) {
   153  	const MICROSECOND = 1000 * time.Nanosecond
   154  	const DAY = 24 * time.Hour
   155  	const WEEK = 7 * DAY
   156  	const YEAR = 365 * DAY
   157  
   158  	for _, tc := range []struct {
   159  		input    string
   160  		expected time.Duration
   161  	}{
   162  		{"1ns", time.Nanosecond},
   163  		{"1s", time.Second},
   164  		{"1us", MICROSECOND},
   165  		{"1m", time.Minute},
   166  		{"1h", time.Hour},
   167  		{"1µs", MICROSECOND},
   168  		{"1y", YEAR},
   169  		{"1w", WEEK},
   170  		{"1d", DAY},
   171  		{"1h15m30.918273645s", time.Hour + 15*time.Minute + 30*time.Second + 918273645*time.Nanosecond},
   172  	} {
   173  		actual, err := parseDuration(tc.input)
   174  
   175  		require.Equal(t, err, nil)
   176  		require.Equal(t, tc.expected, actual)
   177  	}
   178  }