github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/src/query/api/v1/handler/influxdb/write_test.go (about) 1 // Copyright (c) 2019 Uber Technologies, Inc. 2 // 3 // Permission is hereby granted, free of charge, to any person obtaining a copy 4 // of this software and associated documentation files (the "Software"), to deal 5 // in the Software without restriction, including without limitation the rights 6 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 // copies of the Software, and to permit persons to whom the Software is 8 // furnished to do so, subject to the following conditions: 9 // 10 // The above copyright notice and this permission notice shall be included in 11 // all copies or substantial portions of the Software. 12 // 13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 // THE SOFTWARE. 20 21 package influxdb 22 23 import ( 24 "bytes" 25 "compress/gzip" 26 "context" 27 "fmt" 28 "io" 29 "net/http" 30 "net/http/httptest" 31 "testing" 32 "time" 33 34 "github.com/m3db/m3/src/cmd/services/m3coordinator/ingest" 35 "github.com/m3db/m3/src/query/api/v1/options" 36 "github.com/m3db/m3/src/query/models" 37 xtest "github.com/m3db/m3/src/x/test" 38 xtime "github.com/m3db/m3/src/x/time" 39 40 "github.com/golang/mock/gomock" 41 imodels "github.com/influxdata/influxdb/models" 42 "github.com/stretchr/testify/assert" 43 "github.com/stretchr/testify/require" 44 ) 45 46 // human-readable string out of what the iterator produces; 47 // they are easiest for human to handle 48 func (self *ingestIterator) pop(t *testing.T) string { 49 if self.Next() { 50 value := self.Current() 51 assert.Equal(t, 1, len(value.Datapoints)) 52 53 return fmt.Sprintf("%s %v %d", value.Tags.String(), value.Datapoints[0].Value, int64(value.Datapoints[0].Timestamp)) 54 } 55 return "" 56 } 57 58 func TestIngestIterator(t *testing.T) { 59 // test prometheus-illegal measure and label components (should be _s) 60 // as well as all value types influxdb supports 61 s := `?measure:!,?tag1:!=tval1,?tag2:!=tval2 ?key1:!=3,?key2:!=2i 1574838670386469800 62 ?measure:!,?tag1:!=tval1,?tag2:!=tval2 ?key3:!="string",?key4:!=T 1574838670386469801 63 ` 64 points, err := imodels.ParsePoints([]byte(s)) 65 require.NoError(t, err) 66 iter := &ingestIterator{points: points, promRewriter: newPromRewriter()} 67 require.NoError(t, iter.Error()) 68 for _, line := range []string{ 69 "__name__: _measure:___key1:_, _tag1__: tval1, _tag2__: tval2 3 1574838670386469800", 70 "__name__: _measure:___key2:_, _tag1__: tval1, _tag2__: tval2 2 1574838670386469800", 71 "__name__: _measure:___key4:_, _tag1__: tval1, _tag2__: tval2 1 1574838670386469801", 72 "", 73 "", 74 } { 75 assert.Equal(t, line, iter.pop(t)) 76 } 77 require.NoError(t, iter.Error()) 78 } 79 80 func TestIngestIteratorDuplicateTag(t *testing.T) { 81 // Ensure that duplicate tag causes error and no metrics entries 82 s := `measure,lab!=2,lab?=3 key=2i 1574838670386469800 83 ` 84 points, err := imodels.ParsePoints([]byte(s)) 85 require.NoError(t, err) 86 iter := &ingestIterator{points: points, promRewriter: newPromRewriter()} 87 require.NoError(t, iter.Error()) 88 for _, line := range []string{ 89 "", 90 } { 91 assert.Equal(t, line, iter.pop(t)) 92 } 93 require.EqualError(t, iter.Error(), "non-unique Prometheus label lab_") 94 } 95 96 func TestIngestIteratorDuplicateNameTag(t *testing.T) { 97 // Ensure that duplicate name tag causes error and no metrics entries 98 s := `measure,__name__=x key=2i 1574838670386469800 99 ` 100 points, err := imodels.ParsePoints([]byte(s)) 101 require.NoError(t, err) 102 iter := &ingestIterator{points: points, promRewriter: newPromRewriter()} 103 require.NoError(t, iter.Error()) 104 for _, line := range []string{ 105 "", 106 } { 107 assert.Equal(t, line, iter.pop(t)) 108 } 109 require.EqualError(t, iter.Error(), "non-unique Prometheus label __name__") 110 } 111 112 func TestIngestIteratorIssue2125(t *testing.T) { 113 // In the issue, the Tags object is reused across Next()+Current() calls 114 s := `measure,lab=foo k1=1,k2=2 1574838670386469800 115 ` 116 points, err := imodels.ParsePoints([]byte(s)) 117 require.NoError(t, err) 118 119 iter := &ingestIterator{points: points, promRewriter: newPromRewriter()} 120 require.NoError(t, iter.Error()) 121 122 assert.True(t, iter.Next()) 123 value1 := iter.Current() 124 125 assert.True(t, iter.Next()) 126 value2 := iter.Current() 127 require.NoError(t, iter.Error()) 128 129 assert.Equal(t, value1.Tags.String(), "__name__: measure_k1, lab: foo") 130 assert.Equal(t, value2.Tags.String(), "__name__: measure_k2, lab: foo") 131 } 132 133 func TestIngestIteratorWriteTags(t *testing.T) { 134 s := `measure,lab=foo k1=1,k2=2 1574838670386469800 135 ` 136 points, err := imodels.ParsePoints([]byte(s)) 137 require.NoError(t, err) 138 139 writeTags := models.EmptyTags(). 140 AddTag(models.Tag{Name: []byte("lab"), Value: []byte("bar")}). 141 AddTag(models.Tag{Name: []byte("new"), Value: []byte("tag")}) 142 143 iter := &ingestIterator{points: points, promRewriter: newPromRewriter(), writeTags: writeTags} 144 145 assert.True(t, iter.Next()) 146 value1 := iter.Current() 147 require.NoError(t, iter.Error()) 148 149 assert.Equal(t, value1.Tags.String(), "__name__: measure_k1, lab: bar, new: tag") 150 151 assert.True(t, iter.Next()) 152 value2 := iter.Current() 153 require.NoError(t, iter.Error()) 154 155 assert.Equal(t, value2.Tags.String(), "__name__: measure_k2, lab: bar, new: tag") 156 } 157 158 func TestDetermineTimeUnit(t *testing.T) { 159 now := time.Now() 160 zerot := now.Add(time.Duration(-now.UnixNano() % int64(time.Second))) 161 assert.Equal(t, determineTimeUnit(zerot.Add(1*time.Second)), xtime.Second) 162 assert.Equal(t, determineTimeUnit(zerot.Add(2*time.Millisecond)), xtime.Millisecond) 163 assert.Equal(t, determineTimeUnit(zerot.Add(3*time.Microsecond)), xtime.Microsecond) 164 assert.Equal(t, determineTimeUnit(zerot.Add(4*time.Nanosecond)), xtime.Nanosecond) 165 } 166 167 func makeOptions(ds ingest.DownsamplerAndWriter) options.HandlerOptions { 168 return options.EmptyHandlerOptions(). 169 SetDownsamplerAndWriter(ds) 170 } 171 172 func makeInfluxDBLineProtocolMessage(t *testing.T, isGzipped bool, time time.Time, precision time.Duration) io.Reader { 173 t.Helper() 174 ts := fmt.Sprintf("%d", time.UnixNano()/precision.Nanoseconds()) 175 line := fmt.Sprintf("weather,location=us-midwest,season=summer temperature=82 %s", ts) 176 var msg bytes.Buffer 177 if isGzipped { 178 gz := gzip.NewWriter(&msg) 179 _, err := gz.Write([]byte(line)) 180 require.NoError(t, err) 181 err = gz.Close() 182 require.NoError(t, err) 183 } else { 184 msg.WriteString(line) 185 } 186 return bytes.NewReader(msg.Bytes()) 187 } 188 189 func TestInfluxDBWrite(t *testing.T) { 190 type checkWriteBatchFunc func(context.Context, *ingestIterator, ingest.WriteOptions) interface{} 191 192 // small helper for tests where we dont want to check the batch 193 dontCheckWriteBatch := checkWriteBatchFunc( 194 func(context.Context, *ingestIterator, ingest.WriteOptions) interface{} { 195 return nil 196 }, 197 ) 198 199 tests := []struct { 200 name string 201 expectedStatus int 202 requestHeaders map[string]string 203 isGzipped bool 204 checkWriteBatch checkWriteBatchFunc 205 }{ 206 { 207 name: "Gzip Encoded Message", 208 expectedStatus: http.StatusNoContent, 209 isGzipped: true, 210 requestHeaders: map[string]string{ 211 "Content-Encoding": "gzip", 212 }, 213 checkWriteBatch: dontCheckWriteBatch, 214 }, 215 { 216 name: "Wrong Content Encoding", 217 expectedStatus: http.StatusBadRequest, 218 isGzipped: false, 219 requestHeaders: map[string]string{ 220 "Content-Encoding": "gzip", 221 }, 222 checkWriteBatch: dontCheckWriteBatch, 223 }, 224 { 225 name: "Plaintext Message", 226 expectedStatus: http.StatusNoContent, 227 isGzipped: false, 228 requestHeaders: map[string]string{}, 229 checkWriteBatch: dontCheckWriteBatch, 230 }, 231 { 232 name: "Map-Tags-JSON Add Tag", 233 expectedStatus: http.StatusNoContent, 234 isGzipped: false, 235 requestHeaders: map[string]string{ 236 "M3-Map-Tags-JSON": `{"tagMappers": [{"write": {"tag": "t", "value": "v"}}]}`, 237 }, 238 checkWriteBatch: checkWriteBatchFunc( 239 func(_ context.Context, iter *ingestIterator, opts ingest.WriteOptions) interface{} { 240 _, found := iter.writeTags.Get([]byte("t")) 241 require.True(t, found, "tag t will be overwritten") 242 return nil 243 }, 244 ), 245 }, 246 } 247 248 ctrl := xtest.NewController(t) 249 defer ctrl.Finish() 250 251 for _, testCase := range tests { 252 testCase := testCase 253 t.Run(testCase.name, func(tt *testing.T) { 254 mockDownsamplerAndWriter := ingest.NewMockDownsamplerAndWriter(ctrl) 255 // For error reponses we don't expect WriteBatch to be called 256 if testCase.expectedStatus != http.StatusBadRequest { 257 mockDownsamplerAndWriter. 258 EXPECT(). 259 WriteBatch(gomock.Any(), gomock.Any(), gomock.Any()). 260 DoAndReturn(testCase.checkWriteBatch). 261 Times(1) 262 } 263 264 opts := makeOptions(mockDownsamplerAndWriter) 265 handler := NewInfluxWriterHandler(opts) 266 msg := makeInfluxDBLineProtocolMessage(t, testCase.isGzipped, time.Now(), time.Nanosecond) 267 req := httptest.NewRequest(InfluxWriteHTTPMethod, InfluxWriteURL, msg) 268 for header, value := range testCase.requestHeaders { 269 req.Header.Set(header, value) 270 } 271 writer := httptest.NewRecorder() 272 handler.ServeHTTP(writer, req) 273 resp := writer.Result() 274 require.Equal(t, testCase.expectedStatus, resp.StatusCode) 275 resp.Body.Close() 276 }) 277 } 278 } 279 280 func TestInfluxDBWritePrecision(t *testing.T) { 281 tests := []struct { 282 name string 283 expectedStatus int 284 precision string 285 }{ 286 { 287 name: "No precision", 288 expectedStatus: http.StatusNoContent, 289 precision: "", 290 }, 291 { 292 name: "Millisecond precision", 293 expectedStatus: http.StatusNoContent, 294 precision: "ms", 295 }, 296 { 297 name: "Second precision", 298 expectedStatus: http.StatusNoContent, 299 precision: "s", 300 }, 301 } 302 303 ctrl := xtest.NewController(t) 304 defer ctrl.Finish() 305 306 for _, testCase := range tests { 307 testCase := testCase 308 t.Run(testCase.name, func(tt *testing.T) { 309 var precision time.Duration 310 switch testCase.precision { 311 case "": 312 precision = time.Nanosecond 313 case "ms": 314 precision = time.Millisecond 315 case "s": 316 precision = time.Second 317 } 318 319 now := time.Now() 320 321 mockDownsamplerAndWriter := ingest.NewMockDownsamplerAndWriter(ctrl) 322 mockDownsamplerAndWriter. 323 EXPECT(). 324 WriteBatch(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func( 325 _ context.Context, 326 iter *ingestIterator, 327 opts ingest.WriteOptions, 328 ) interface{} { 329 require.Equal(tt, now.Truncate(precision).UnixNano(), iter.points[0].UnixNano(), "correct precision") 330 return nil 331 }).Times(1) 332 333 opts := makeOptions(mockDownsamplerAndWriter) 334 handler := NewInfluxWriterHandler(opts) 335 336 msg := makeInfluxDBLineProtocolMessage(t, false, now, precision) 337 var url string 338 if testCase.precision == "" { 339 url = InfluxWriteURL 340 } else { 341 url = InfluxWriteURL + fmt.Sprintf("?precision=%s", testCase.precision) 342 } 343 req := httptest.NewRequest(InfluxWriteHTTPMethod, url, msg) 344 writer := httptest.NewRecorder() 345 handler.ServeHTTP(writer, req) 346 resp := writer.Result() 347 require.Equal(t, testCase.expectedStatus, resp.StatusCode) 348 resp.Body.Close() 349 }) 350 } 351 }