github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/src/query/api/v1/handler/influxdb/write_test.go (about)

     1  // Copyright (c) 2019 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package influxdb
    22  
    23  import (
    24  	"bytes"
    25  	"compress/gzip"
    26  	"context"
    27  	"fmt"
    28  	"io"
    29  	"net/http"
    30  	"net/http/httptest"
    31  	"testing"
    32  	"time"
    33  
    34  	"github.com/m3db/m3/src/cmd/services/m3coordinator/ingest"
    35  	"github.com/m3db/m3/src/query/api/v1/options"
    36  	"github.com/m3db/m3/src/query/models"
    37  	xtest "github.com/m3db/m3/src/x/test"
    38  	xtime "github.com/m3db/m3/src/x/time"
    39  
    40  	"github.com/golang/mock/gomock"
    41  	imodels "github.com/influxdata/influxdb/models"
    42  	"github.com/stretchr/testify/assert"
    43  	"github.com/stretchr/testify/require"
    44  )
    45  
    46  // human-readable string out of what the iterator produces;
    47  // they are easiest for human to handle
    48  func (self *ingestIterator) pop(t *testing.T) string {
    49  	if self.Next() {
    50  		value := self.Current()
    51  		assert.Equal(t, 1, len(value.Datapoints))
    52  
    53  		return fmt.Sprintf("%s %v %d", value.Tags.String(), value.Datapoints[0].Value, int64(value.Datapoints[0].Timestamp))
    54  	}
    55  	return ""
    56  }
    57  
    58  func TestIngestIterator(t *testing.T) {
    59  	// test prometheus-illegal measure and label components (should be _s)
    60  	// as well as all value types influxdb supports
    61  	s := `?measure:!,?tag1:!=tval1,?tag2:!=tval2 ?key1:!=3,?key2:!=2i 1574838670386469800
    62  ?measure:!,?tag1:!=tval1,?tag2:!=tval2 ?key3:!="string",?key4:!=T 1574838670386469801
    63  `
    64  	points, err := imodels.ParsePoints([]byte(s))
    65  	require.NoError(t, err)
    66  	iter := &ingestIterator{points: points, promRewriter: newPromRewriter()}
    67  	require.NoError(t, iter.Error())
    68  	for _, line := range []string{
    69  		"__name__: _measure:___key1:_, _tag1__: tval1, _tag2__: tval2 3 1574838670386469800",
    70  		"__name__: _measure:___key2:_, _tag1__: tval1, _tag2__: tval2 2 1574838670386469800",
    71  		"__name__: _measure:___key4:_, _tag1__: tval1, _tag2__: tval2 1 1574838670386469801",
    72  		"",
    73  		"",
    74  	} {
    75  		assert.Equal(t, line, iter.pop(t))
    76  	}
    77  	require.NoError(t, iter.Error())
    78  }
    79  
    80  func TestIngestIteratorDuplicateTag(t *testing.T) {
    81  	// Ensure that duplicate tag causes error and no metrics entries
    82  	s := `measure,lab!=2,lab?=3 key=2i 1574838670386469800
    83  `
    84  	points, err := imodels.ParsePoints([]byte(s))
    85  	require.NoError(t, err)
    86  	iter := &ingestIterator{points: points, promRewriter: newPromRewriter()}
    87  	require.NoError(t, iter.Error())
    88  	for _, line := range []string{
    89  		"",
    90  	} {
    91  		assert.Equal(t, line, iter.pop(t))
    92  	}
    93  	require.EqualError(t, iter.Error(), "non-unique Prometheus label lab_")
    94  }
    95  
    96  func TestIngestIteratorDuplicateNameTag(t *testing.T) {
    97  	// Ensure that duplicate name tag causes error and no metrics entries
    98  	s := `measure,__name__=x key=2i 1574838670386469800
    99  `
   100  	points, err := imodels.ParsePoints([]byte(s))
   101  	require.NoError(t, err)
   102  	iter := &ingestIterator{points: points, promRewriter: newPromRewriter()}
   103  	require.NoError(t, iter.Error())
   104  	for _, line := range []string{
   105  		"",
   106  	} {
   107  		assert.Equal(t, line, iter.pop(t))
   108  	}
   109  	require.EqualError(t, iter.Error(), "non-unique Prometheus label __name__")
   110  }
   111  
   112  func TestIngestIteratorIssue2125(t *testing.T) {
   113  	// In the issue, the Tags object is reused across Next()+Current() calls
   114  	s := `measure,lab=foo k1=1,k2=2 1574838670386469800
   115  `
   116  	points, err := imodels.ParsePoints([]byte(s))
   117  	require.NoError(t, err)
   118  
   119  	iter := &ingestIterator{points: points, promRewriter: newPromRewriter()}
   120  	require.NoError(t, iter.Error())
   121  
   122  	assert.True(t, iter.Next())
   123  	value1 := iter.Current()
   124  
   125  	assert.True(t, iter.Next())
   126  	value2 := iter.Current()
   127  	require.NoError(t, iter.Error())
   128  
   129  	assert.Equal(t, value1.Tags.String(), "__name__: measure_k1, lab: foo")
   130  	assert.Equal(t, value2.Tags.String(), "__name__: measure_k2, lab: foo")
   131  }
   132  
   133  func TestIngestIteratorWriteTags(t *testing.T) {
   134  	s := `measure,lab=foo k1=1,k2=2 1574838670386469800
   135  `
   136  	points, err := imodels.ParsePoints([]byte(s))
   137  	require.NoError(t, err)
   138  
   139  	writeTags := models.EmptyTags().
   140  		AddTag(models.Tag{Name: []byte("lab"), Value: []byte("bar")}).
   141  		AddTag(models.Tag{Name: []byte("new"), Value: []byte("tag")})
   142  
   143  	iter := &ingestIterator{points: points, promRewriter: newPromRewriter(), writeTags: writeTags}
   144  
   145  	assert.True(t, iter.Next())
   146  	value1 := iter.Current()
   147  	require.NoError(t, iter.Error())
   148  
   149  	assert.Equal(t, value1.Tags.String(), "__name__: measure_k1, lab: bar, new: tag")
   150  
   151  	assert.True(t, iter.Next())
   152  	value2 := iter.Current()
   153  	require.NoError(t, iter.Error())
   154  
   155  	assert.Equal(t, value2.Tags.String(), "__name__: measure_k2, lab: bar, new: tag")
   156  }
   157  
   158  func TestDetermineTimeUnit(t *testing.T) {
   159  	now := time.Now()
   160  	zerot := now.Add(time.Duration(-now.UnixNano() % int64(time.Second)))
   161  	assert.Equal(t, determineTimeUnit(zerot.Add(1*time.Second)), xtime.Second)
   162  	assert.Equal(t, determineTimeUnit(zerot.Add(2*time.Millisecond)), xtime.Millisecond)
   163  	assert.Equal(t, determineTimeUnit(zerot.Add(3*time.Microsecond)), xtime.Microsecond)
   164  	assert.Equal(t, determineTimeUnit(zerot.Add(4*time.Nanosecond)), xtime.Nanosecond)
   165  }
   166  
   167  func makeOptions(ds ingest.DownsamplerAndWriter) options.HandlerOptions {
   168  	return options.EmptyHandlerOptions().
   169  		SetDownsamplerAndWriter(ds)
   170  }
   171  
   172  func makeInfluxDBLineProtocolMessage(t *testing.T, isGzipped bool, time time.Time, precision time.Duration) io.Reader {
   173  	t.Helper()
   174  	ts := fmt.Sprintf("%d", time.UnixNano()/precision.Nanoseconds())
   175  	line := fmt.Sprintf("weather,location=us-midwest,season=summer temperature=82 %s", ts)
   176  	var msg bytes.Buffer
   177  	if isGzipped {
   178  		gz := gzip.NewWriter(&msg)
   179  		_, err := gz.Write([]byte(line))
   180  		require.NoError(t, err)
   181  		err = gz.Close()
   182  		require.NoError(t, err)
   183  	} else {
   184  		msg.WriteString(line)
   185  	}
   186  	return bytes.NewReader(msg.Bytes())
   187  }
   188  
   189  func TestInfluxDBWrite(t *testing.T) {
   190  	type checkWriteBatchFunc func(context.Context, *ingestIterator, ingest.WriteOptions) interface{}
   191  
   192  	// small helper for tests where we dont want to check the batch
   193  	dontCheckWriteBatch := checkWriteBatchFunc(
   194  		func(context.Context, *ingestIterator, ingest.WriteOptions) interface{} {
   195  			return nil
   196  		},
   197  	)
   198  
   199  	tests := []struct {
   200  		name            string
   201  		expectedStatus  int
   202  		requestHeaders  map[string]string
   203  		isGzipped       bool
   204  		checkWriteBatch checkWriteBatchFunc
   205  	}{
   206  		{
   207  			name:           "Gzip Encoded Message",
   208  			expectedStatus: http.StatusNoContent,
   209  			isGzipped:      true,
   210  			requestHeaders: map[string]string{
   211  				"Content-Encoding": "gzip",
   212  			},
   213  			checkWriteBatch: dontCheckWriteBatch,
   214  		},
   215  		{
   216  			name:           "Wrong Content Encoding",
   217  			expectedStatus: http.StatusBadRequest,
   218  			isGzipped:      false,
   219  			requestHeaders: map[string]string{
   220  				"Content-Encoding": "gzip",
   221  			},
   222  			checkWriteBatch: dontCheckWriteBatch,
   223  		},
   224  		{
   225  			name:            "Plaintext Message",
   226  			expectedStatus:  http.StatusNoContent,
   227  			isGzipped:       false,
   228  			requestHeaders:  map[string]string{},
   229  			checkWriteBatch: dontCheckWriteBatch,
   230  		},
   231  		{
   232  			name:           "Map-Tags-JSON Add Tag",
   233  			expectedStatus: http.StatusNoContent,
   234  			isGzipped:      false,
   235  			requestHeaders: map[string]string{
   236  				"M3-Map-Tags-JSON": `{"tagMappers": [{"write": {"tag": "t", "value": "v"}}]}`,
   237  			},
   238  			checkWriteBatch: checkWriteBatchFunc(
   239  				func(_ context.Context, iter *ingestIterator, opts ingest.WriteOptions) interface{} {
   240  					_, found := iter.writeTags.Get([]byte("t"))
   241  					require.True(t, found, "tag t will be overwritten")
   242  					return nil
   243  				},
   244  			),
   245  		},
   246  	}
   247  
   248  	ctrl := xtest.NewController(t)
   249  	defer ctrl.Finish()
   250  
   251  	for _, testCase := range tests {
   252  		testCase := testCase
   253  		t.Run(testCase.name, func(tt *testing.T) {
   254  			mockDownsamplerAndWriter := ingest.NewMockDownsamplerAndWriter(ctrl)
   255  			// For error reponses we don't expect WriteBatch to be called
   256  			if testCase.expectedStatus != http.StatusBadRequest {
   257  				mockDownsamplerAndWriter.
   258  					EXPECT().
   259  					WriteBatch(gomock.Any(), gomock.Any(), gomock.Any()).
   260  					DoAndReturn(testCase.checkWriteBatch).
   261  					Times(1)
   262  			}
   263  
   264  			opts := makeOptions(mockDownsamplerAndWriter)
   265  			handler := NewInfluxWriterHandler(opts)
   266  			msg := makeInfluxDBLineProtocolMessage(t, testCase.isGzipped, time.Now(), time.Nanosecond)
   267  			req := httptest.NewRequest(InfluxWriteHTTPMethod, InfluxWriteURL, msg)
   268  			for header, value := range testCase.requestHeaders {
   269  				req.Header.Set(header, value)
   270  			}
   271  			writer := httptest.NewRecorder()
   272  			handler.ServeHTTP(writer, req)
   273  			resp := writer.Result()
   274  			require.Equal(t, testCase.expectedStatus, resp.StatusCode)
   275  			resp.Body.Close()
   276  		})
   277  	}
   278  }
   279  
   280  func TestInfluxDBWritePrecision(t *testing.T) {
   281  	tests := []struct {
   282  		name           string
   283  		expectedStatus int
   284  		precision      string
   285  	}{
   286  		{
   287  			name:           "No precision",
   288  			expectedStatus: http.StatusNoContent,
   289  			precision:      "",
   290  		},
   291  		{
   292  			name:           "Millisecond precision",
   293  			expectedStatus: http.StatusNoContent,
   294  			precision:      "ms",
   295  		},
   296  		{
   297  			name:           "Second precision",
   298  			expectedStatus: http.StatusNoContent,
   299  			precision:      "s",
   300  		},
   301  	}
   302  
   303  	ctrl := xtest.NewController(t)
   304  	defer ctrl.Finish()
   305  
   306  	for _, testCase := range tests {
   307  		testCase := testCase
   308  		t.Run(testCase.name, func(tt *testing.T) {
   309  			var precision time.Duration
   310  			switch testCase.precision {
   311  			case "":
   312  				precision = time.Nanosecond
   313  			case "ms":
   314  				precision = time.Millisecond
   315  			case "s":
   316  				precision = time.Second
   317  			}
   318  
   319  			now := time.Now()
   320  
   321  			mockDownsamplerAndWriter := ingest.NewMockDownsamplerAndWriter(ctrl)
   322  			mockDownsamplerAndWriter.
   323  				EXPECT().
   324  				WriteBatch(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(
   325  				_ context.Context,
   326  				iter *ingestIterator,
   327  				opts ingest.WriteOptions,
   328  			) interface{} {
   329  				require.Equal(tt, now.Truncate(precision).UnixNano(), iter.points[0].UnixNano(), "correct precision")
   330  				return nil
   331  			}).Times(1)
   332  
   333  			opts := makeOptions(mockDownsamplerAndWriter)
   334  			handler := NewInfluxWriterHandler(opts)
   335  
   336  			msg := makeInfluxDBLineProtocolMessage(t, false, now, precision)
   337  			var url string
   338  			if testCase.precision == "" {
   339  				url = InfluxWriteURL
   340  			} else {
   341  				url = InfluxWriteURL + fmt.Sprintf("?precision=%s", testCase.precision)
   342  			}
   343  			req := httptest.NewRequest(InfluxWriteHTTPMethod, url, msg)
   344  			writer := httptest.NewRecorder()
   345  			handler.ServeHTTP(writer, req)
   346  			resp := writer.Result()
   347  			require.Equal(t, testCase.expectedStatus, resp.StatusCode)
   348  			resp.Body.Close()
   349  		})
   350  	}
   351  }