github.com/m3db/m3@v1.5.0/src/query/api/v1/handler/graphite/find_test.go (about) 1 // Copyright (c) 2019 Uber Technologies, Inc. 2 // 3 // Permission is hereby granted, free of charge, to any person obtaining a copy 4 // of this software and associated documentation files (the "Software"), to deal 5 // in the Software without restriction, including without limitation the rights 6 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 // copies of the Software, and to permit persons to whom the Software is 8 // furnished to do so, subject to the following conditions: 9 // 10 // The above copyright notice and this permission notice shall be included in 11 // all copies or substantial portions of the Software. 12 // 13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 // THE SOFTWARE. 20 21 package graphite 22 23 import ( 24 "bytes" 25 "encoding/json" 26 "fmt" 27 "net/http" 28 "net/url" 29 "sort" 30 "strings" 31 "testing" 32 "time" 33 34 "github.com/golang/mock/gomock" 35 "github.com/stretchr/testify/assert" 36 "github.com/stretchr/testify/require" 37 38 "github.com/m3db/m3/src/m3ninx/doc" 39 "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" 40 "github.com/m3db/m3/src/query/api/v1/options" 41 "github.com/m3db/m3/src/query/block" 42 "github.com/m3db/m3/src/query/graphite/graphite" 43 "github.com/m3db/m3/src/query/models" 44 "github.com/m3db/m3/src/query/storage" 45 "github.com/m3db/m3/src/query/storage/m3/consolidators" 46 "github.com/m3db/m3/src/x/headers" 47 xtest "github.com/m3db/m3/src/x/test" 48 xtime "github.com/m3db/m3/src/x/time" 49 ) 50 51 // dates is a tuple of a date with a valid string representation 52 type date struct { 53 t xtime.UnixNano 54 s string 55 } 56 57 var ( 58 from = date{ 59 s: "14:38_20150618", 60 t: xtime.ToUnixNano(time.Date(2015, time.June, 18, 14, 38, 0, 0, time.UTC)), 61 } 62 until = date{ 63 s: "1432581620", 64 t: xtime.ToUnixNano(time.Date(2015, time.May, 25, 19, 20, 20, 0, time.UTC)), 65 } 66 ) 67 68 type completeTagQueryMatcher struct { 69 matchers []models.Matcher 70 filterNameTagsIndexStart int 71 filterNameTagsIndexEnd int 72 } 73 74 func (m *completeTagQueryMatcher) String() string { 75 q := storage.CompleteTagsQuery{ 76 TagMatchers: m.matchers, 77 } 78 return q.String() 79 } 80 81 func (m *completeTagQueryMatcher) Matches(x interface{}) bool { 82 q, ok := x.(*storage.CompleteTagsQuery) 83 if !ok { 84 return false 85 } 86 87 if !q.Start.Equal(from.t) { 88 return false 89 } 90 91 if !q.End.Equal(until.t) { 92 return false 93 } 94 95 if q.CompleteNameOnly { 96 return false 97 } 98 99 if m.filterNameTagsIndexStart == 0 && m.filterNameTagsIndexEnd == 0 { 100 // Default query completing single graphite path index value. 101 if len(q.FilterNameTags) != 1 { 102 return false 103 } 104 105 // Both queries should filter on __g1__. 106 if !bytes.Equal(q.FilterNameTags[0], []byte("__g1__")) { 107 return false 108 } 109 } else { 110 // Unterminated query completing many grapth path index values. 111 n := m.filterNameTagsIndexEnd 112 expected := make([][]byte, 0, n) 113 for i := m.filterNameTagsIndexStart; i < m.filterNameTagsIndexEnd; i++ { 114 expected = append(expected, graphite.TagName(i)) 115 } 116 117 if len(q.FilterNameTags) != len(expected) { 118 return false 119 } 120 121 for i := range expected { 122 if !bytes.Equal(q.FilterNameTags[i], expected[i]) { 123 return false 124 } 125 } 126 } 127 128 if len(q.TagMatchers) != len(m.matchers) { 129 return false 130 } 131 132 for i, qMatcher := range q.TagMatchers { 133 if !bytes.Equal(qMatcher.Name, m.matchers[i].Name) { 134 return false 135 } 136 if !bytes.Equal(qMatcher.Value, m.matchers[i].Value) { 137 return false 138 } 139 if qMatcher.Type != m.matchers[i].Type { 140 return false 141 } 142 } 143 144 return true 145 } 146 147 var _ gomock.Matcher = &completeTagQueryMatcher{} 148 149 func b(s string) []byte { return []byte(s) } 150 func bs(ss ...string) [][]byte { 151 bb := make([][]byte, len(ss)) 152 for i, s := range ss { 153 bb[i] = b(s) 154 } 155 return bb 156 } 157 158 type writer struct { 159 results []string 160 header http.Header 161 } 162 163 var _ http.ResponseWriter = &writer{} 164 165 func (w *writer) WriteHeader(_ int) {} 166 func (w *writer) Header() http.Header { 167 if w.header == nil { 168 w.header = make(http.Header) 169 } 170 171 return w.header 172 } 173 174 func (w *writer) Write(b []byte) (int, error) { 175 if w.results == nil { 176 w.results = make([]string, 0, 10) 177 } 178 179 w.results = append(w.results, string(b)) 180 return len(b), nil 181 } 182 183 type result struct { 184 ID string `json:"id"` 185 Text string `json:"text"` 186 Leaf int `json:"leaf"` 187 Expandable int `json:"expandable"` 188 AllowChildren int `json:"allowChildren"` 189 } 190 191 type results []result 192 193 func (r results) Len() int { return len(r) } 194 func (r results) Swap(i, j int) { r[i], r[j] = r[j], r[i] } 195 func (r results) Less(i, j int) bool { 196 return strings.Compare(r[i].ID, r[j].ID) == -1 197 } 198 199 func makeNoChildrenResult(id, text string) result { 200 return result{ 201 ID: id, 202 Text: text, 203 Leaf: 1, 204 Expandable: 0, 205 AllowChildren: 0, 206 } 207 } 208 209 func makeWithChildrenResult(id, text string) result { 210 return result{ 211 ID: id, 212 Text: text, 213 Leaf: 0, 214 Expandable: 1, 215 AllowChildren: 1, 216 } 217 } 218 219 type limitTest struct { 220 name string 221 ex, ex2 bool 222 header string 223 } 224 225 var ( 226 bothCompleteLimitTest = limitTest{"both complete", true, true, ""} 227 limitTests = []limitTest{ 228 bothCompleteLimitTest, 229 { 230 "both incomplete", false, false, 231 fmt.Sprintf("%s,%s_%s", headers.LimitHeaderSeriesLimitApplied, "foo", "bar"), 232 }, 233 { 234 "with terminator incomplete", true, false, 235 "foo_bar", 236 }, 237 { 238 "with children incomplete", false, true, 239 headers.LimitHeaderSeriesLimitApplied, 240 }, 241 } 242 ) 243 244 func TestFind(t *testing.T) { 245 for _, httpMethod := range FindHTTPMethods { 246 testFind(t, testFindOptions{ 247 httpMethod: httpMethod, 248 }) 249 } 250 } 251 252 type testFindOptions struct { 253 httpMethod string 254 } 255 256 type testFindQuery struct { 257 expectMatchers *completeTagQueryMatcher 258 mockResult func(lt limitTest) *consolidators.CompleteTagsResult 259 } 260 261 func testFind(t *testing.T, opts testFindOptions) { 262 warningsFooBar := block.Warnings{ 263 block.Warning{ 264 Name: "foo", 265 Message: "bar", 266 }, 267 } 268 269 for _, test := range []struct { 270 query string 271 limitTests []limitTest 272 terminatedQuery *testFindQuery 273 childQuery *testFindQuery 274 expectedResultsWithoutExpandableAndLeafDuplicates results 275 expectedResultsWithExpandableAndLeafDuplicates results 276 }{ 277 { 278 query: "foo.b*", 279 limitTests: limitTests, 280 terminatedQuery: &testFindQuery{ 281 expectMatchers: &completeTagQueryMatcher{ 282 matchers: []models.Matcher{ 283 {Type: models.MatchEqual, Name: b("__g0__"), Value: b("foo")}, 284 {Type: models.MatchRegexp, Name: b("__g1__"), Value: b(`b[^\.]*`)}, 285 {Type: models.MatchNotField, Name: b("__g2__")}, 286 }, 287 }, 288 mockResult: func(lt limitTest) *consolidators.CompleteTagsResult { 289 return &consolidators.CompleteTagsResult{ 290 CompleteNameOnly: false, 291 CompletedTags: []consolidators.CompletedTag{ 292 {Name: b("__g1__"), Values: bs("bug", "bar", "baz")}, 293 }, 294 Metadata: block.ResultMetadata{ 295 LocalOnly: true, 296 Exhaustive: lt.ex, 297 }, 298 } 299 }, 300 }, 301 childQuery: &testFindQuery{ 302 expectMatchers: &completeTagQueryMatcher{ 303 matchers: []models.Matcher{ 304 {Type: models.MatchEqual, Name: b("__g0__"), Value: b("foo")}, 305 {Type: models.MatchRegexp, Name: b("__g1__"), Value: b(`b[^\.]*`)}, 306 {Type: models.MatchField, Name: b("__g2__")}, 307 }, 308 }, 309 mockResult: func(lt limitTest) *consolidators.CompleteTagsResult { 310 var warnings block.Warnings 311 if !lt.ex2 { 312 warnings = warningsFooBar 313 } 314 return &consolidators.CompleteTagsResult{ 315 CompleteNameOnly: false, 316 CompletedTags: []consolidators.CompletedTag{ 317 {Name: b("__g1__"), Values: bs("baz", "bix", "bug")}, 318 }, 319 Metadata: block.ResultMetadata{ 320 LocalOnly: false, 321 Exhaustive: true, 322 Warnings: warnings, 323 }, 324 } 325 }, 326 }, 327 expectedResultsWithoutExpandableAndLeafDuplicates: results{ 328 makeNoChildrenResult("foo.bar", "bar"), 329 makeWithChildrenResult("foo.baz", "baz"), 330 makeWithChildrenResult("foo.bix", "bix"), 331 makeWithChildrenResult("foo.bug", "bug"), 332 }, 333 expectedResultsWithExpandableAndLeafDuplicates: results{ 334 makeNoChildrenResult("foo.bar", "bar"), 335 makeNoChildrenResult("foo.baz", "baz"), 336 makeWithChildrenResult("foo.baz", "baz"), 337 makeWithChildrenResult("foo.bix", "bix"), 338 makeNoChildrenResult("foo.bug", "bug"), 339 makeWithChildrenResult("foo.bug", "bug"), 340 }, 341 }, 342 { 343 query: "foo.**.*", 344 childQuery: &testFindQuery{ 345 expectMatchers: &completeTagQueryMatcher{ 346 matchers: []models.Matcher{ 347 { 348 Type: models.MatchRegexp, 349 Name: b("__g0__"), Value: b(".*"), 350 }, 351 { 352 Type: models.MatchRegexp, 353 Name: doc.IDReservedFieldName, 354 Value: b(`foo\.+.*[^\.]*`), 355 }, 356 }, 357 filterNameTagsIndexStart: 2, 358 filterNameTagsIndexEnd: 102, 359 }, 360 mockResult: func(_ limitTest) *consolidators.CompleteTagsResult { 361 return &consolidators.CompleteTagsResult{ 362 CompleteNameOnly: false, 363 CompletedTags: []consolidators.CompletedTag{ 364 {Name: b("__g2__"), Values: bs("bar0", "bar1")}, 365 {Name: b("__g3__"), Values: bs("baz0", "baz1", "baz2")}, 366 }, 367 Metadata: block.ResultMetadata{ 368 LocalOnly: true, 369 Exhaustive: true, 370 }, 371 } 372 }, 373 }, 374 expectedResultsWithoutExpandableAndLeafDuplicates: results{ 375 makeWithChildrenResult("foo.**.bar0", "bar0"), 376 makeWithChildrenResult("foo.**.bar1", "bar1"), 377 makeWithChildrenResult("foo.**.baz0", "baz0"), 378 makeWithChildrenResult("foo.**.baz1", "baz1"), 379 makeWithChildrenResult("foo.**.baz2", "baz2"), 380 }, 381 expectedResultsWithExpandableAndLeafDuplicates: results{ 382 makeWithChildrenResult("foo.**.bar0", "bar0"), 383 makeWithChildrenResult("foo.**.bar1", "bar1"), 384 makeWithChildrenResult("foo.**.baz0", "baz0"), 385 makeWithChildrenResult("foo.**.baz1", "baz1"), 386 makeWithChildrenResult("foo.**.baz2", "baz2"), 387 }, 388 }, 389 } { 390 // Set which limit tests should be performed for this query. 391 testCaseLimitTests := test.limitTests 392 if len(limitTests) == 0 { 393 // Just test case where both are complete. 394 testCaseLimitTests = []limitTest{bothCompleteLimitTest} 395 } 396 397 type testVariation struct { 398 limitTest limitTest 399 includeBothExpandableAndLeaf bool 400 expectedResults results 401 } 402 403 var testVarations []testVariation 404 for _, limitTest := range testCaseLimitTests { 405 testVarations = append(testVarations, 406 // Test case with default find result options. 407 testVariation{ 408 limitTest: limitTest, 409 includeBothExpandableAndLeaf: false, 410 expectedResults: test.expectedResultsWithoutExpandableAndLeafDuplicates, 411 }, 412 // Test case test for overloaded find result options. 413 testVariation{ 414 limitTest: limitTest, 415 includeBothExpandableAndLeaf: true, 416 expectedResults: test.expectedResultsWithExpandableAndLeafDuplicates, 417 }) 418 } 419 420 for _, variation := range testVarations { 421 // nolint: govet 422 limitTest := variation.limitTest 423 includeBothExpandableAndLeaf := variation.includeBothExpandableAndLeaf 424 expectedResults := variation.expectedResults 425 t.Run(fmt.Sprintf("%s-%s", test.query, limitTest.name), func(t *testing.T) { 426 ctrl := xtest.NewController(t) 427 defer ctrl.Finish() 428 429 store := storage.NewMockStorage(ctrl) 430 431 if q := test.terminatedQuery; q != nil { 432 // Set up no children case. 433 store.EXPECT(). 434 CompleteTags(gomock.Any(), q.expectMatchers, gomock.Any()). 435 Return(q.mockResult(limitTest), nil) 436 } 437 438 if q := test.childQuery; q != nil { 439 // Set up children case. 440 store.EXPECT(). 441 CompleteTags(gomock.Any(), q.expectMatchers, gomock.Any()). 442 Return(q.mockResult(limitTest), nil) 443 } 444 445 builder, err := handleroptions.NewFetchOptionsBuilder( 446 handleroptions.FetchOptionsBuilderOptions{ 447 Timeout: 15 * time.Second, 448 }) 449 require.NoError(t, err) 450 451 handlerOpts := options.EmptyHandlerOptions(). 452 SetGraphiteFindFetchOptionsBuilder(builder). 453 SetStorage(store) 454 // Set the relevant result options and save back to handler options. 455 graphiteStorageOpts := handlerOpts.GraphiteStorageOptions() 456 graphiteStorageOpts.FindResultsIncludeBothExpandableAndLeaf = includeBothExpandableAndLeaf 457 handlerOpts = handlerOpts.SetGraphiteStorageOptions(graphiteStorageOpts) 458 459 h := NewFindHandler(handlerOpts) 460 461 // Execute the query. 462 params := make(url.Values) 463 params.Set("query", test.query) 464 params.Set("from", from.s) 465 params.Set("until", until.s) 466 467 w := &writer{} 468 req := &http.Request{Method: opts.httpMethod} 469 switch opts.httpMethod { 470 case http.MethodGet: 471 req.URL = &url.URL{ 472 RawQuery: params.Encode(), 473 } 474 case http.MethodPost: 475 req.Form = params 476 } 477 478 h.ServeHTTP(w, req) 479 480 // Convert results to comparable format. 481 require.Equal(t, 1, len(w.results)) 482 r := make(results, 0) 483 decoder := json.NewDecoder(bytes.NewBufferString((w.results[0]))) 484 require.NoError(t, decoder.Decode(&r)) 485 sort.Sort(r) 486 487 require.Equal(t, expectedResults, r) 488 actual := w.Header().Get(headers.LimitHeader) 489 assert.Equal(t, limitTest.header, actual) 490 }) 491 } 492 } 493 }