github.com/m3db/m3@v1.5.0/src/query/api/v1/handler/graphite/find_test.go (about)

     1  // Copyright (c) 2019 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package graphite
    22  
    23  import (
    24  	"bytes"
    25  	"encoding/json"
    26  	"fmt"
    27  	"net/http"
    28  	"net/url"
    29  	"sort"
    30  	"strings"
    31  	"testing"
    32  	"time"
    33  
    34  	"github.com/golang/mock/gomock"
    35  	"github.com/stretchr/testify/assert"
    36  	"github.com/stretchr/testify/require"
    37  
    38  	"github.com/m3db/m3/src/m3ninx/doc"
    39  	"github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions"
    40  	"github.com/m3db/m3/src/query/api/v1/options"
    41  	"github.com/m3db/m3/src/query/block"
    42  	"github.com/m3db/m3/src/query/graphite/graphite"
    43  	"github.com/m3db/m3/src/query/models"
    44  	"github.com/m3db/m3/src/query/storage"
    45  	"github.com/m3db/m3/src/query/storage/m3/consolidators"
    46  	"github.com/m3db/m3/src/x/headers"
    47  	xtest "github.com/m3db/m3/src/x/test"
    48  	xtime "github.com/m3db/m3/src/x/time"
    49  )
    50  
    51  // dates is a tuple of a date with a valid string representation
    52  type date struct {
    53  	t xtime.UnixNano
    54  	s string
    55  }
    56  
    57  var (
    58  	from = date{
    59  		s: "14:38_20150618",
    60  		t: xtime.ToUnixNano(time.Date(2015, time.June, 18, 14, 38, 0, 0, time.UTC)),
    61  	}
    62  	until = date{
    63  		s: "1432581620",
    64  		t: xtime.ToUnixNano(time.Date(2015, time.May, 25, 19, 20, 20, 0, time.UTC)),
    65  	}
    66  )
    67  
    68  type completeTagQueryMatcher struct {
    69  	matchers                 []models.Matcher
    70  	filterNameTagsIndexStart int
    71  	filterNameTagsIndexEnd   int
    72  }
    73  
    74  func (m *completeTagQueryMatcher) String() string {
    75  	q := storage.CompleteTagsQuery{
    76  		TagMatchers: m.matchers,
    77  	}
    78  	return q.String()
    79  }
    80  
    81  func (m *completeTagQueryMatcher) Matches(x interface{}) bool {
    82  	q, ok := x.(*storage.CompleteTagsQuery)
    83  	if !ok {
    84  		return false
    85  	}
    86  
    87  	if !q.Start.Equal(from.t) {
    88  		return false
    89  	}
    90  
    91  	if !q.End.Equal(until.t) {
    92  		return false
    93  	}
    94  
    95  	if q.CompleteNameOnly {
    96  		return false
    97  	}
    98  
    99  	if m.filterNameTagsIndexStart == 0 && m.filterNameTagsIndexEnd == 0 {
   100  		// Default query completing single graphite path index value.
   101  		if len(q.FilterNameTags) != 1 {
   102  			return false
   103  		}
   104  
   105  		// Both queries should filter on __g1__.
   106  		if !bytes.Equal(q.FilterNameTags[0], []byte("__g1__")) {
   107  			return false
   108  		}
   109  	} else {
   110  		// Unterminated query completing many grapth path index values.
   111  		n := m.filterNameTagsIndexEnd
   112  		expected := make([][]byte, 0, n)
   113  		for i := m.filterNameTagsIndexStart; i < m.filterNameTagsIndexEnd; i++ {
   114  			expected = append(expected, graphite.TagName(i))
   115  		}
   116  
   117  		if len(q.FilterNameTags) != len(expected) {
   118  			return false
   119  		}
   120  
   121  		for i := range expected {
   122  			if !bytes.Equal(q.FilterNameTags[i], expected[i]) {
   123  				return false
   124  			}
   125  		}
   126  	}
   127  
   128  	if len(q.TagMatchers) != len(m.matchers) {
   129  		return false
   130  	}
   131  
   132  	for i, qMatcher := range q.TagMatchers {
   133  		if !bytes.Equal(qMatcher.Name, m.matchers[i].Name) {
   134  			return false
   135  		}
   136  		if !bytes.Equal(qMatcher.Value, m.matchers[i].Value) {
   137  			return false
   138  		}
   139  		if qMatcher.Type != m.matchers[i].Type {
   140  			return false
   141  		}
   142  	}
   143  
   144  	return true
   145  }
   146  
   147  var _ gomock.Matcher = &completeTagQueryMatcher{}
   148  
   149  func b(s string) []byte { return []byte(s) }
   150  func bs(ss ...string) [][]byte {
   151  	bb := make([][]byte, len(ss))
   152  	for i, s := range ss {
   153  		bb[i] = b(s)
   154  	}
   155  	return bb
   156  }
   157  
   158  type writer struct {
   159  	results []string
   160  	header  http.Header
   161  }
   162  
   163  var _ http.ResponseWriter = &writer{}
   164  
   165  func (w *writer) WriteHeader(_ int) {}
   166  func (w *writer) Header() http.Header {
   167  	if w.header == nil {
   168  		w.header = make(http.Header)
   169  	}
   170  
   171  	return w.header
   172  }
   173  
   174  func (w *writer) Write(b []byte) (int, error) {
   175  	if w.results == nil {
   176  		w.results = make([]string, 0, 10)
   177  	}
   178  
   179  	w.results = append(w.results, string(b))
   180  	return len(b), nil
   181  }
   182  
   183  type result struct {
   184  	ID            string `json:"id"`
   185  	Text          string `json:"text"`
   186  	Leaf          int    `json:"leaf"`
   187  	Expandable    int    `json:"expandable"`
   188  	AllowChildren int    `json:"allowChildren"`
   189  }
   190  
   191  type results []result
   192  
   193  func (r results) Len() int      { return len(r) }
   194  func (r results) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
   195  func (r results) Less(i, j int) bool {
   196  	return strings.Compare(r[i].ID, r[j].ID) == -1
   197  }
   198  
   199  func makeNoChildrenResult(id, text string) result {
   200  	return result{
   201  		ID:            id,
   202  		Text:          text,
   203  		Leaf:          1,
   204  		Expandable:    0,
   205  		AllowChildren: 0,
   206  	}
   207  }
   208  
   209  func makeWithChildrenResult(id, text string) result {
   210  	return result{
   211  		ID:            id,
   212  		Text:          text,
   213  		Leaf:          0,
   214  		Expandable:    1,
   215  		AllowChildren: 1,
   216  	}
   217  }
   218  
   219  type limitTest struct {
   220  	name    string
   221  	ex, ex2 bool
   222  	header  string
   223  }
   224  
   225  var (
   226  	bothCompleteLimitTest = limitTest{"both complete", true, true, ""}
   227  	limitTests            = []limitTest{
   228  		bothCompleteLimitTest,
   229  		{
   230  			"both incomplete", false, false,
   231  			fmt.Sprintf("%s,%s_%s", headers.LimitHeaderSeriesLimitApplied, "foo", "bar"),
   232  		},
   233  		{
   234  			"with terminator incomplete", true, false,
   235  			"foo_bar",
   236  		},
   237  		{
   238  			"with children incomplete", false, true,
   239  			headers.LimitHeaderSeriesLimitApplied,
   240  		},
   241  	}
   242  )
   243  
   244  func TestFind(t *testing.T) {
   245  	for _, httpMethod := range FindHTTPMethods {
   246  		testFind(t, testFindOptions{
   247  			httpMethod: httpMethod,
   248  		})
   249  	}
   250  }
   251  
   252  type testFindOptions struct {
   253  	httpMethod string
   254  }
   255  
   256  type testFindQuery struct {
   257  	expectMatchers *completeTagQueryMatcher
   258  	mockResult     func(lt limitTest) *consolidators.CompleteTagsResult
   259  }
   260  
   261  func testFind(t *testing.T, opts testFindOptions) {
   262  	warningsFooBar := block.Warnings{
   263  		block.Warning{
   264  			Name:    "foo",
   265  			Message: "bar",
   266  		},
   267  	}
   268  
   269  	for _, test := range []struct {
   270  		query                                             string
   271  		limitTests                                        []limitTest
   272  		terminatedQuery                                   *testFindQuery
   273  		childQuery                                        *testFindQuery
   274  		expectedResultsWithoutExpandableAndLeafDuplicates results
   275  		expectedResultsWithExpandableAndLeafDuplicates    results
   276  	}{
   277  		{
   278  			query:      "foo.b*",
   279  			limitTests: limitTests,
   280  			terminatedQuery: &testFindQuery{
   281  				expectMatchers: &completeTagQueryMatcher{
   282  					matchers: []models.Matcher{
   283  						{Type: models.MatchEqual, Name: b("__g0__"), Value: b("foo")},
   284  						{Type: models.MatchRegexp, Name: b("__g1__"), Value: b(`b[^\.]*`)},
   285  						{Type: models.MatchNotField, Name: b("__g2__")},
   286  					},
   287  				},
   288  				mockResult: func(lt limitTest) *consolidators.CompleteTagsResult {
   289  					return &consolidators.CompleteTagsResult{
   290  						CompleteNameOnly: false,
   291  						CompletedTags: []consolidators.CompletedTag{
   292  							{Name: b("__g1__"), Values: bs("bug", "bar", "baz")},
   293  						},
   294  						Metadata: block.ResultMetadata{
   295  							LocalOnly:  true,
   296  							Exhaustive: lt.ex,
   297  						},
   298  					}
   299  				},
   300  			},
   301  			childQuery: &testFindQuery{
   302  				expectMatchers: &completeTagQueryMatcher{
   303  					matchers: []models.Matcher{
   304  						{Type: models.MatchEqual, Name: b("__g0__"), Value: b("foo")},
   305  						{Type: models.MatchRegexp, Name: b("__g1__"), Value: b(`b[^\.]*`)},
   306  						{Type: models.MatchField, Name: b("__g2__")},
   307  					},
   308  				},
   309  				mockResult: func(lt limitTest) *consolidators.CompleteTagsResult {
   310  					var warnings block.Warnings
   311  					if !lt.ex2 {
   312  						warnings = warningsFooBar
   313  					}
   314  					return &consolidators.CompleteTagsResult{
   315  						CompleteNameOnly: false,
   316  						CompletedTags: []consolidators.CompletedTag{
   317  							{Name: b("__g1__"), Values: bs("baz", "bix", "bug")},
   318  						},
   319  						Metadata: block.ResultMetadata{
   320  							LocalOnly:  false,
   321  							Exhaustive: true,
   322  							Warnings:   warnings,
   323  						},
   324  					}
   325  				},
   326  			},
   327  			expectedResultsWithoutExpandableAndLeafDuplicates: results{
   328  				makeNoChildrenResult("foo.bar", "bar"),
   329  				makeWithChildrenResult("foo.baz", "baz"),
   330  				makeWithChildrenResult("foo.bix", "bix"),
   331  				makeWithChildrenResult("foo.bug", "bug"),
   332  			},
   333  			expectedResultsWithExpandableAndLeafDuplicates: results{
   334  				makeNoChildrenResult("foo.bar", "bar"),
   335  				makeNoChildrenResult("foo.baz", "baz"),
   336  				makeWithChildrenResult("foo.baz", "baz"),
   337  				makeWithChildrenResult("foo.bix", "bix"),
   338  				makeNoChildrenResult("foo.bug", "bug"),
   339  				makeWithChildrenResult("foo.bug", "bug"),
   340  			},
   341  		},
   342  		{
   343  			query: "foo.**.*",
   344  			childQuery: &testFindQuery{
   345  				expectMatchers: &completeTagQueryMatcher{
   346  					matchers: []models.Matcher{
   347  						{
   348  							Type: models.MatchRegexp,
   349  							Name: b("__g0__"), Value: b(".*"),
   350  						},
   351  						{
   352  							Type:  models.MatchRegexp,
   353  							Name:  doc.IDReservedFieldName,
   354  							Value: b(`foo\.+.*[^\.]*`),
   355  						},
   356  					},
   357  					filterNameTagsIndexStart: 2,
   358  					filterNameTagsIndexEnd:   102,
   359  				},
   360  				mockResult: func(_ limitTest) *consolidators.CompleteTagsResult {
   361  					return &consolidators.CompleteTagsResult{
   362  						CompleteNameOnly: false,
   363  						CompletedTags: []consolidators.CompletedTag{
   364  							{Name: b("__g2__"), Values: bs("bar0", "bar1")},
   365  							{Name: b("__g3__"), Values: bs("baz0", "baz1", "baz2")},
   366  						},
   367  						Metadata: block.ResultMetadata{
   368  							LocalOnly:  true,
   369  							Exhaustive: true,
   370  						},
   371  					}
   372  				},
   373  			},
   374  			expectedResultsWithoutExpandableAndLeafDuplicates: results{
   375  				makeWithChildrenResult("foo.**.bar0", "bar0"),
   376  				makeWithChildrenResult("foo.**.bar1", "bar1"),
   377  				makeWithChildrenResult("foo.**.baz0", "baz0"),
   378  				makeWithChildrenResult("foo.**.baz1", "baz1"),
   379  				makeWithChildrenResult("foo.**.baz2", "baz2"),
   380  			},
   381  			expectedResultsWithExpandableAndLeafDuplicates: results{
   382  				makeWithChildrenResult("foo.**.bar0", "bar0"),
   383  				makeWithChildrenResult("foo.**.bar1", "bar1"),
   384  				makeWithChildrenResult("foo.**.baz0", "baz0"),
   385  				makeWithChildrenResult("foo.**.baz1", "baz1"),
   386  				makeWithChildrenResult("foo.**.baz2", "baz2"),
   387  			},
   388  		},
   389  	} {
   390  		// Set which limit tests should be performed for this query.
   391  		testCaseLimitTests := test.limitTests
   392  		if len(limitTests) == 0 {
   393  			// Just test case where both are complete.
   394  			testCaseLimitTests = []limitTest{bothCompleteLimitTest}
   395  		}
   396  
   397  		type testVariation struct {
   398  			limitTest                    limitTest
   399  			includeBothExpandableAndLeaf bool
   400  			expectedResults              results
   401  		}
   402  
   403  		var testVarations []testVariation
   404  		for _, limitTest := range testCaseLimitTests {
   405  			testVarations = append(testVarations,
   406  				// Test case with default find result options.
   407  				testVariation{
   408  					limitTest:                    limitTest,
   409  					includeBothExpandableAndLeaf: false,
   410  					expectedResults:              test.expectedResultsWithoutExpandableAndLeafDuplicates,
   411  				},
   412  				// Test case test for overloaded find result options.
   413  				testVariation{
   414  					limitTest:                    limitTest,
   415  					includeBothExpandableAndLeaf: true,
   416  					expectedResults:              test.expectedResultsWithExpandableAndLeafDuplicates,
   417  				})
   418  		}
   419  
   420  		for _, variation := range testVarations {
   421  			// nolint: govet
   422  			limitTest := variation.limitTest
   423  			includeBothExpandableAndLeaf := variation.includeBothExpandableAndLeaf
   424  			expectedResults := variation.expectedResults
   425  			t.Run(fmt.Sprintf("%s-%s", test.query, limitTest.name), func(t *testing.T) {
   426  				ctrl := xtest.NewController(t)
   427  				defer ctrl.Finish()
   428  
   429  				store := storage.NewMockStorage(ctrl)
   430  
   431  				if q := test.terminatedQuery; q != nil {
   432  					// Set up no children case.
   433  					store.EXPECT().
   434  						CompleteTags(gomock.Any(), q.expectMatchers, gomock.Any()).
   435  						Return(q.mockResult(limitTest), nil)
   436  				}
   437  
   438  				if q := test.childQuery; q != nil {
   439  					// Set up children case.
   440  					store.EXPECT().
   441  						CompleteTags(gomock.Any(), q.expectMatchers, gomock.Any()).
   442  						Return(q.mockResult(limitTest), nil)
   443  				}
   444  
   445  				builder, err := handleroptions.NewFetchOptionsBuilder(
   446  					handleroptions.FetchOptionsBuilderOptions{
   447  						Timeout: 15 * time.Second,
   448  					})
   449  				require.NoError(t, err)
   450  
   451  				handlerOpts := options.EmptyHandlerOptions().
   452  					SetGraphiteFindFetchOptionsBuilder(builder).
   453  					SetStorage(store)
   454  				// Set the relevant result options and save back to handler options.
   455  				graphiteStorageOpts := handlerOpts.GraphiteStorageOptions()
   456  				graphiteStorageOpts.FindResultsIncludeBothExpandableAndLeaf = includeBothExpandableAndLeaf
   457  				handlerOpts = handlerOpts.SetGraphiteStorageOptions(graphiteStorageOpts)
   458  
   459  				h := NewFindHandler(handlerOpts)
   460  
   461  				// Execute the query.
   462  				params := make(url.Values)
   463  				params.Set("query", test.query)
   464  				params.Set("from", from.s)
   465  				params.Set("until", until.s)
   466  
   467  				w := &writer{}
   468  				req := &http.Request{Method: opts.httpMethod}
   469  				switch opts.httpMethod {
   470  				case http.MethodGet:
   471  					req.URL = &url.URL{
   472  						RawQuery: params.Encode(),
   473  					}
   474  				case http.MethodPost:
   475  					req.Form = params
   476  				}
   477  
   478  				h.ServeHTTP(w, req)
   479  
   480  				// Convert results to comparable format.
   481  				require.Equal(t, 1, len(w.results))
   482  				r := make(results, 0)
   483  				decoder := json.NewDecoder(bytes.NewBufferString((w.results[0])))
   484  				require.NoError(t, decoder.Decode(&r))
   485  				sort.Sort(r)
   486  
   487  				require.Equal(t, expectedResults, r)
   488  				actual := w.Header().Get(headers.LimitHeader)
   489  				assert.Equal(t, limitTest.header, actual)
   490  			})
   491  		}
   492  	}
   493  }