github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/logql/syntax/lex_test.go (about) 1 package syntax 2 3 import ( 4 "strings" 5 "testing" 6 "text/scanner" 7 "time" 8 9 "github.com/stretchr/testify/require" 10 ) 11 12 func TestLex(t *testing.T) { 13 for _, tc := range []struct { 14 input string 15 expected []int 16 }{ 17 {`{foo="bar"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, 18 {"{foo=\"bar\"} |~ `\\w+`", []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING}}, 19 {`{foo="bar"} |~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING}}, 20 {`{foo="bar"} |~ "\\w+" | latency > 250ms`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, DURATION}}, 21 {`{foo="bar"} |~ "\\w+" | foo = 0ms`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, EQ, DURATION}}, 22 {`{foo="bar"} |~ "\\w+" | latency > 1h15m30.918273645s`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, DURATION}}, 23 {`{foo="bar"} |~ "\\w+" | latency > 1h0.0m0s`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, DURATION}}, 24 { 25 `{foo="bar"} |~ "\\w+" | latency > 1h0.0m0s or foo == 4.00 and bar ="foo"`, 26 []int{ 27 OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, 28 PIPE, IDENTIFIER, GT, DURATION, OR, IDENTIFIER, CMP_EQ, NUMBER, AND, IDENTIFIER, EQ, STRING, 29 }, 30 }, 31 { 32 `{foo="bar"} |~ "\\w+" | duration > 1h0.0m0s or avg == 4.00 and bar ="foo"`, 33 []int{ 34 OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, 35 PIPE, IDENTIFIER, GT, DURATION, OR, IDENTIFIER, CMP_EQ, NUMBER, AND, IDENTIFIER, EQ, STRING, 36 }, 37 }, 38 { 39 `{foo="bar"} |~ "\\w+" | latency > 1h0.0m0s or foo == 4.00 and bar ="foo" | unwrap foo`, 40 []int{ 41 OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, 42 PIPE, IDENTIFIER, GT, DURATION, OR, IDENTIFIER, CMP_EQ, NUMBER, AND, IDENTIFIER, EQ, STRING, PIPE, UNWRAP, IDENTIFIER, 43 }, 44 }, 45 {`{foo="bar"} |~ "\\w+" | size > 250kB`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, BYTES}}, 46 { 47 `{foo="bar"} |~ "\\w+" | size > 250kB and latency <= 1h15m30s or bar=1`, 48 []int{ 49 OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, 50 IDENTIFIER, GT, BYTES, AND, IDENTIFIER, LTE, DURATION, OR, IDENTIFIER, EQ, NUMBER, 51 }, 52 }, 53 { 54 `{foo="bar"} |~ "\\w+" | size > 200MiB or foo == 4.00`, 55 []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, IDENTIFIER, GT, BYTES, OR, IDENTIFIER, CMP_EQ, NUMBER}, 56 }, 57 {`{ foo = "bar" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, 58 {`{ foo != "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NEQ, STRING, CLOSE_BRACE}}, 59 {`{ foo =~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, RE, STRING, CLOSE_BRACE}}, 60 {`{ foo !~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NRE, STRING, CLOSE_BRACE}}, 61 {`{ foo = "bar", bar != "baz" }`, []int{ 62 OPEN_BRACE, IDENTIFIER, EQ, STRING, 63 COMMA, IDENTIFIER, NEQ, STRING, CLOSE_BRACE, 64 }}, 65 {`{ foo = "ba\"r" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, 66 {`rate({foo="bar"}[10s])`, []int{RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS}}, 67 {`rate_counter({foo="bar"} | unwrap foo[10s])`, []int{RATE_COUNTER, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, UNWRAP, IDENTIFIER, RANGE, CLOSE_PARENTHESIS}}, 68 {`count_over_time({foo="bar"}[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS}}, 69 {`count_over_time({foo="bar"} |~ "\\w+" | unwrap foo[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING, PIPE, UNWRAP, IDENTIFIER, RANGE, CLOSE_PARENTHESIS}}, 70 {`sum(count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, 71 {`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, NUMBER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, 72 {`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, NUMBER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}}, 73 {`sum(max(rate({foo="bar"}[5m])) by (foo,bar)) by (foo)`, []int{SUM, OPEN_PARENTHESIS, MAX, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, CLOSE_PARENTHESIS}}, 74 {`{foo="bar"} #|~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, 75 {`#{foo="bar"} |~ "\\w+"`, []int{}}, 76 {`{foo="#"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, 77 {`{foo="bar"}|logfmt|ip="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}}, 78 {`{foo="bar"}|logfmt|rate="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}}, 79 {`{foo="bar"}|logfmt|b=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}}, 80 {`{foo="bar"}|logfmt|=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE_EXACT, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}}, 81 {`ip`, []int{IDENTIFIER}}, 82 {`rate`, []int{IDENTIFIER}}, 83 {`{foo="bar"} | json | baz="#"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, PIPE, IDENTIFIER, EQ, STRING}}, 84 {`{foo="bar"} 85 # |~ "\\w+" 86 | json`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON}}, 87 {`{foo="bar"} | json code="response.code", param="request.params[0]"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, IDENTIFIER, EQ, STRING, COMMA, IDENTIFIER, EQ, STRING}}, 88 } { 89 t.Run(tc.input, func(t *testing.T) { 90 actual := []int{} 91 l := lexer{ 92 Scanner: scanner.Scanner{ 93 Mode: scanner.SkipComments | scanner.ScanStrings, 94 }, 95 } 96 l.Init(strings.NewReader(tc.input)) 97 var lval exprSymType 98 for { 99 tok := l.Lex(&lval) 100 if tok == 0 { 101 break 102 } 103 actual = append(actual, tok) 104 } 105 require.Equal(t, tc.expected, actual) 106 }) 107 } 108 } 109 110 func Test_isFunction(t *testing.T) { 111 tests := []struct { 112 next string 113 want bool 114 }{ 115 {" (", true}, 116 {"(", true}, 117 {"by (", true}, 118 {"by(", true}, 119 {"by (", true}, 120 {" by (", true}, 121 {" by(", true}, 122 {"by (", true}, 123 {"without (", true}, 124 {"without(", true}, 125 {"without (", true}, 126 {" without (", true}, 127 {" without(", true}, 128 {"without (", true}, 129 {" ( whatever is this", true}, 130 {" (foo,bar)", true}, 131 {"\r\n \t\t\r\n \n (foo,bar)", true}, 132 133 {" foo (", false}, 134 {"123", false}, 135 {"", false}, 136 {" ", false}, 137 {" )(", false}, 138 {"byfoo", false}, 139 {"without foo", false}, 140 } 141 for _, tt := range tests { 142 t.Run(tt.next, func(t *testing.T) { 143 sc := scanner.Scanner{} 144 sc.Init(strings.NewReader(tt.next)) 145 if got := isFunction(sc); got != tt.want { 146 t.Errorf("isFunction() = %v, want %v", got, tt.want) 147 } 148 }) 149 } 150 } 151 152 func Test_parseDuration(t *testing.T) { 153 const MICROSECOND = 1000 * time.Nanosecond 154 const DAY = 24 * time.Hour 155 const WEEK = 7 * DAY 156 const YEAR = 365 * DAY 157 158 for _, tc := range []struct { 159 input string 160 expected time.Duration 161 }{ 162 {"1ns", time.Nanosecond}, 163 {"1s", time.Second}, 164 {"1us", MICROSECOND}, 165 {"1m", time.Minute}, 166 {"1h", time.Hour}, 167 {"1µs", MICROSECOND}, 168 {"1y", YEAR}, 169 {"1w", WEEK}, 170 {"1d", DAY}, 171 {"1h15m30.918273645s", time.Hour + 15*time.Minute + 30*time.Second + 918273645*time.Nanosecond}, 172 } { 173 actual, err := parseDuration(tc.input) 174 175 require.Equal(t, err, nil) 176 require.Equal(t, tc.expected, actual) 177 } 178 }