github.com/m3db/m3@v1.5.0/src/metrics/encoding/protobuf/unaggregated_iterator_test.go (about) 1 // Copyright (c) 2018 Uber Technologies, Inc. 2 // 3 // Permission is hereby granted, free of charge, to any person obtaining a copy 4 // of this software and associated documentation files (the "Software"), to deal 5 // in the Software without restriction, including without limitation the rights 6 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 // copies of the Software, and to permit persons to whom the Software is 8 // furnished to do so, subject to the following conditions: 9 // 10 // The above copyright notice and this permission notice shall be included in 11 // all copies or substantial portions of the Software. 12 // 13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 // THE SOFTWARE. 20 21 package protobuf 22 23 import ( 24 "bytes" 25 "encoding/binary" 26 "io" 27 "strings" 28 "testing" 29 30 "github.com/m3db/m3/src/metrics/encoding" 31 "github.com/m3db/m3/src/metrics/metric/aggregated" 32 "github.com/m3db/m3/src/metrics/metric/unaggregated" 33 34 "github.com/google/go-cmp/cmp" 35 "github.com/stretchr/testify/require" 36 ) 37 38 func TestUnaggregatedIteratorDecodeCounterWithMetadatas(t *testing.T) { 39 inputs := []unaggregated.CounterWithMetadatas{ 40 { 41 Counter: testCounter1, 42 StagedMetadatas: testStagedMetadatas1, 43 }, 44 { 45 Counter: testCounter2, 46 StagedMetadatas: testStagedMetadatas1, 47 }, 48 { 49 Counter: testCounter1, 50 StagedMetadatas: testStagedMetadatas2, 51 }, 52 { 53 Counter: testCounter2, 54 StagedMetadatas: testStagedMetadatas2, 55 }, 56 } 57 58 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 59 for _, input := range inputs { 60 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 61 Type: encoding.CounterWithMetadatasType, 62 CounterWithMetadatas: input, 63 })) 64 } 65 dataBuf := enc.Relinquish() 66 defer dataBuf.Close() 67 68 var ( 69 i int 70 stream = bytes.NewReader(dataBuf.Bytes()) 71 ) 72 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 73 defer it.Close() 74 for it.Next() { 75 res := it.Current() 76 require.Equal(t, encoding.CounterWithMetadatasType, res.Type) 77 require.Equal(t, inputs[i], res.CounterWithMetadatas) 78 i++ 79 } 80 require.Equal(t, io.EOF, it.Err()) 81 require.Equal(t, len(inputs), i) 82 } 83 84 func TestUnaggregatedIteratorDecodeBatchTimerWithMetadatas(t *testing.T) { 85 inputs := []unaggregated.BatchTimerWithMetadatas{ 86 { 87 BatchTimer: testBatchTimer1, 88 StagedMetadatas: testStagedMetadatas1, 89 }, 90 { 91 BatchTimer: testBatchTimer2, 92 StagedMetadatas: testStagedMetadatas1, 93 }, 94 { 95 BatchTimer: testBatchTimer1, 96 StagedMetadatas: testStagedMetadatas2, 97 }, 98 { 99 BatchTimer: testBatchTimer2, 100 StagedMetadatas: testStagedMetadatas2, 101 }, 102 } 103 104 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 105 for _, input := range inputs { 106 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 107 Type: encoding.BatchTimerWithMetadatasType, 108 BatchTimerWithMetadatas: input, 109 })) 110 } 111 dataBuf := enc.Relinquish() 112 defer dataBuf.Close() 113 114 var ( 115 i int 116 stream = bytes.NewReader(dataBuf.Bytes()) 117 ) 118 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 119 defer it.Close() 120 for it.Next() { 121 res := it.Current() 122 require.Equal(t, encoding.BatchTimerWithMetadatasType, res.Type) 123 require.Equal(t, inputs[i], res.BatchTimerWithMetadatas) 124 i++ 125 } 126 require.Equal(t, io.EOF, it.Err()) 127 require.Equal(t, len(inputs), i) 128 } 129 130 func TestUnaggregatedIteratorDecodeGaugeWithMetadatas(t *testing.T) { 131 inputs := []unaggregated.GaugeWithMetadatas{ 132 { 133 Gauge: testGauge1, 134 StagedMetadatas: testStagedMetadatas1, 135 }, 136 { 137 Gauge: testGauge2, 138 StagedMetadatas: testStagedMetadatas1, 139 }, 140 { 141 Gauge: testGauge1, 142 StagedMetadatas: testStagedMetadatas2, 143 }, 144 { 145 Gauge: testGauge2, 146 StagedMetadatas: testStagedMetadatas2, 147 }, 148 } 149 150 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 151 for _, input := range inputs { 152 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 153 Type: encoding.GaugeWithMetadatasType, 154 GaugeWithMetadatas: input, 155 })) 156 } 157 dataBuf := enc.Relinquish() 158 defer dataBuf.Close() 159 160 var ( 161 i int 162 stream = bytes.NewReader(dataBuf.Bytes()) 163 ) 164 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 165 defer it.Close() 166 for it.Next() { 167 res := it.Current() 168 require.Equal(t, encoding.GaugeWithMetadatasType, res.Type) 169 require.Equal(t, inputs[i], res.GaugeWithMetadatas) 170 i++ 171 } 172 require.Equal(t, io.EOF, it.Err()) 173 require.Equal(t, len(inputs), i) 174 } 175 176 func TestUnaggregatedIteratorDecodeForwardedMetricWithMetadata(t *testing.T) { 177 inputs := []aggregated.ForwardedMetricWithMetadata{ 178 { 179 ForwardedMetric: testForwardedMetric1, 180 ForwardMetadata: testForwardMetadata1, 181 }, 182 { 183 ForwardedMetric: testForwardedMetric2, 184 ForwardMetadata: testForwardMetadata1, 185 }, 186 { 187 ForwardedMetric: testForwardedMetric1, 188 ForwardMetadata: testForwardMetadata2, 189 }, 190 { 191 ForwardedMetric: testForwardedMetric2, 192 ForwardMetadata: testForwardMetadata2, 193 }, 194 } 195 196 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 197 for _, input := range inputs { 198 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 199 Type: encoding.ForwardedMetricWithMetadataType, 200 ForwardedMetricWithMetadata: input, 201 })) 202 } 203 dataBuf := enc.Relinquish() 204 defer dataBuf.Close() 205 206 var ( 207 i int 208 stream = bytes.NewReader(dataBuf.Bytes()) 209 ) 210 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 211 defer it.Close() 212 for it.Next() { 213 res := it.Current() 214 require.Equal(t, encoding.ForwardedMetricWithMetadataType, res.Type) 215 require.Equal(t, inputs[i], res.ForwardedMetricWithMetadata) 216 i++ 217 } 218 require.Equal(t, io.EOF, it.Err()) 219 require.Equal(t, len(inputs), i) 220 } 221 func TestUnaggregatedIteratorDecodePassthroughMetricWithMetadata(t *testing.T) { 222 inputs := []aggregated.PassthroughMetricWithMetadata{ 223 { 224 Metric: testPassthroughMetric1, 225 StoragePolicy: testPassthroughMetadata1, 226 }, 227 { 228 Metric: testPassthroughMetric2, 229 StoragePolicy: testPassthroughMetadata1, 230 }, 231 { 232 Metric: testPassthroughMetric1, 233 StoragePolicy: testPassthroughMetadata2, 234 }, 235 { 236 Metric: testPassthroughMetric2, 237 StoragePolicy: testPassthroughMetadata2, 238 }, 239 } 240 241 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 242 for _, input := range inputs { 243 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 244 Type: encoding.PassthroughMetricWithMetadataType, 245 PassthroughMetricWithMetadata: input, 246 })) 247 } 248 dataBuf := enc.Relinquish() 249 defer dataBuf.Close() 250 251 var ( 252 i int 253 stream = bytes.NewReader(dataBuf.Bytes()) 254 ) 255 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 256 defer it.Close() 257 for it.Next() { 258 res := it.Current() 259 require.Equal(t, encoding.PassthroughMetricWithMetadataType, res.Type) 260 require.Equal(t, inputs[i], res.PassthroughMetricWithMetadata) 261 i++ 262 } 263 require.Equal(t, io.EOF, it.Err()) 264 require.Equal(t, len(inputs), i) 265 } 266 267 func TestUnaggregatedIteratorDecodeTimedMetricWithMetadata(t *testing.T) { 268 inputs := []aggregated.TimedMetricWithMetadata{ 269 { 270 Metric: testTimedMetric1, 271 TimedMetadata: testTimedMetadata1, 272 }, 273 { 274 Metric: testTimedMetric2, 275 TimedMetadata: testTimedMetadata1, 276 }, 277 { 278 Metric: testTimedMetric1, 279 TimedMetadata: testTimedMetadata2, 280 }, 281 { 282 Metric: testTimedMetric2, 283 TimedMetadata: testTimedMetadata2, 284 }, 285 } 286 287 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 288 for _, input := range inputs { 289 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 290 Type: encoding.TimedMetricWithMetadataType, 291 TimedMetricWithMetadata: input, 292 })) 293 } 294 dataBuf := enc.Relinquish() 295 defer dataBuf.Close() 296 297 var ( 298 i int 299 stream = bytes.NewReader(dataBuf.Bytes()) 300 ) 301 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 302 defer it.Close() 303 for it.Next() { 304 res := it.Current() 305 require.Equal(t, encoding.TimedMetricWithMetadataType, res.Type) 306 require.Equal(t, inputs[i], res.TimedMetricWithMetadata) 307 i++ 308 } 309 require.Equal(t, io.EOF, it.Err()) 310 require.Equal(t, len(inputs), i) 311 } 312 313 func TestUnaggregatedIteratorDecodeStress(t *testing.T) { 314 inputs := []interface{}{ 315 unaggregated.CounterWithMetadatas{ 316 Counter: testCounter1, 317 StagedMetadatas: testStagedMetadatas1, 318 }, 319 unaggregated.BatchTimerWithMetadatas{ 320 BatchTimer: testBatchTimer1, 321 StagedMetadatas: testStagedMetadatas1, 322 }, 323 unaggregated.GaugeWithMetadatas{ 324 Gauge: testGauge1, 325 StagedMetadatas: testStagedMetadatas1, 326 }, 327 aggregated.ForwardedMetricWithMetadata{ 328 ForwardedMetric: testForwardedMetric1, 329 ForwardMetadata: testForwardMetadata1, 330 }, 331 aggregated.TimedMetricWithMetadata{ 332 Metric: testTimedMetric1, 333 TimedMetadata: testTimedMetadata1, 334 }, 335 aggregated.PassthroughMetricWithMetadata{ 336 Metric: testPassthroughMetric1, 337 StoragePolicy: testPassthroughMetadata1, 338 }, 339 unaggregated.CounterWithMetadatas{ 340 Counter: testCounter2, 341 StagedMetadatas: testStagedMetadatas1, 342 }, 343 unaggregated.BatchTimerWithMetadatas{ 344 BatchTimer: testBatchTimer2, 345 StagedMetadatas: testStagedMetadatas1, 346 }, 347 unaggregated.GaugeWithMetadatas{ 348 Gauge: testGauge2, 349 StagedMetadatas: testStagedMetadatas1, 350 }, 351 aggregated.ForwardedMetricWithMetadata{ 352 ForwardedMetric: testForwardedMetric2, 353 ForwardMetadata: testForwardMetadata1, 354 }, 355 unaggregated.CounterWithMetadatas{ 356 Counter: testCounter1, 357 StagedMetadatas: testStagedMetadatas2, 358 }, 359 unaggregated.BatchTimerWithMetadatas{ 360 BatchTimer: testBatchTimer1, 361 StagedMetadatas: testStagedMetadatas2, 362 }, 363 unaggregated.GaugeWithMetadatas{ 364 Gauge: testGauge1, 365 StagedMetadatas: testStagedMetadatas2, 366 }, 367 aggregated.ForwardedMetricWithMetadata{ 368 ForwardedMetric: testForwardedMetric1, 369 ForwardMetadata: testForwardMetadata2, 370 }, 371 unaggregated.CounterWithMetadatas{ 372 Counter: testCounter2, 373 StagedMetadatas: testStagedMetadatas2, 374 }, 375 unaggregated.BatchTimerWithMetadatas{ 376 BatchTimer: testBatchTimer2, 377 StagedMetadatas: testStagedMetadatas2, 378 }, 379 unaggregated.GaugeWithMetadatas{ 380 Gauge: testGauge2, 381 StagedMetadatas: testStagedMetadatas2, 382 }, 383 aggregated.ForwardedMetricWithMetadata{ 384 ForwardedMetric: testForwardedMetric2, 385 ForwardMetadata: testForwardMetadata2, 386 }, 387 aggregated.TimedMetricWithMetadata{ 388 Metric: testTimedMetric2, 389 TimedMetadata: testTimedMetadata2, 390 }, 391 aggregated.PassthroughMetricWithMetadata{ 392 Metric: testPassthroughMetric2, 393 StoragePolicy: testPassthroughMetadata2, 394 }, 395 } 396 397 numIter := 1000 398 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 399 for iter := 0; iter < numIter; iter++ { 400 for _, input := range inputs { 401 var msg encoding.UnaggregatedMessageUnion 402 switch input := input.(type) { 403 case unaggregated.CounterWithMetadatas: 404 msg = encoding.UnaggregatedMessageUnion{ 405 Type: encoding.CounterWithMetadatasType, 406 CounterWithMetadatas: input, 407 } 408 case unaggregated.BatchTimerWithMetadatas: 409 msg = encoding.UnaggregatedMessageUnion{ 410 Type: encoding.BatchTimerWithMetadatasType, 411 BatchTimerWithMetadatas: input, 412 } 413 case unaggregated.GaugeWithMetadatas: 414 msg = encoding.UnaggregatedMessageUnion{ 415 Type: encoding.GaugeWithMetadatasType, 416 GaugeWithMetadatas: input, 417 } 418 case aggregated.ForwardedMetricWithMetadata: 419 msg = encoding.UnaggregatedMessageUnion{ 420 Type: encoding.ForwardedMetricWithMetadataType, 421 ForwardedMetricWithMetadata: input, 422 } 423 case aggregated.TimedMetricWithMetadata: 424 msg = encoding.UnaggregatedMessageUnion{ 425 Type: encoding.TimedMetricWithMetadataType, 426 TimedMetricWithMetadata: input, 427 } 428 case aggregated.PassthroughMetricWithMetadata: 429 msg = encoding.UnaggregatedMessageUnion{ 430 Type: encoding.PassthroughMetricWithMetadataType, 431 PassthroughMetricWithMetadata: input, 432 } 433 default: 434 require.Fail(t, "unrecognized type %T", input) 435 } 436 require.NoError(t, enc.EncodeMessage(msg)) 437 } 438 } 439 dataBuf := enc.Relinquish() 440 defer dataBuf.Close() 441 442 var ( 443 i int 444 stream = bytes.NewReader(dataBuf.Bytes()) 445 ) 446 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 447 defer it.Close() 448 for it.Next() { 449 res := it.Current() 450 j := i % len(inputs) 451 switch expectedRes := inputs[j].(type) { 452 case unaggregated.CounterWithMetadatas: 453 require.Equal(t, encoding.CounterWithMetadatasType, res.Type) 454 require.True(t, cmp.Equal(expectedRes, res.CounterWithMetadatas, testCmpOpts...)) 455 case unaggregated.BatchTimerWithMetadatas: 456 require.Equal(t, encoding.BatchTimerWithMetadatasType, res.Type) 457 require.True(t, cmp.Equal(expectedRes, res.BatchTimerWithMetadatas, testCmpOpts...)) 458 case unaggregated.GaugeWithMetadatas: 459 require.Equal(t, encoding.GaugeWithMetadatasType, res.Type) 460 require.True(t, cmp.Equal(expectedRes, res.GaugeWithMetadatas, testCmpOpts...)) 461 case aggregated.ForwardedMetricWithMetadata: 462 require.Equal(t, encoding.ForwardedMetricWithMetadataType, res.Type) 463 require.True(t, cmp.Equal(expectedRes, res.ForwardedMetricWithMetadata, testCmpOpts...)) 464 case aggregated.TimedMetricWithMetadata: 465 require.Equal(t, encoding.TimedMetricWithMetadataType, res.Type) 466 require.True(t, cmp.Equal(expectedRes, res.TimedMetricWithMetadata, testCmpOpts...)) 467 case aggregated.PassthroughMetricWithMetadata: 468 require.Equal(t, encoding.PassthroughMetricWithMetadataType, res.Type) 469 require.True(t, cmp.Equal(expectedRes, res.PassthroughMetricWithMetadata, testCmpOpts...)) 470 default: 471 require.Fail(t, "unknown input type: %T", inputs[j]) 472 } 473 i++ 474 } 475 require.Equal(t, io.EOF, it.Err()) 476 require.Equal(t, len(inputs)*numIter, i) 477 } 478 479 func TestUnaggregatedIteratorMessageTooLarge(t *testing.T) { 480 input := unaggregated.GaugeWithMetadatas{ 481 Gauge: testGauge1, 482 StagedMetadatas: testStagedMetadatas1, 483 } 484 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 485 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 486 Type: encoding.GaugeWithMetadatasType, 487 GaugeWithMetadatas: input, 488 })) 489 dataBuf := enc.Relinquish() 490 defer dataBuf.Close() 491 492 var ( 493 i int 494 stream = bytes.NewReader(dataBuf.Bytes()) 495 ) 496 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions().SetMaxMessageSize(1)) 497 defer it.Close() 498 for it.Next() { 499 i++ 500 } 501 err := it.Err() 502 require.Error(t, err) 503 require.True(t, strings.Contains(err.Error(), "larger than supported max message size")) 504 require.Equal(t, 0, i) 505 } 506 507 func TestUnaggregatedIteratorNextOnError(t *testing.T) { 508 input := unaggregated.GaugeWithMetadatas{ 509 Gauge: testGauge1, 510 StagedMetadatas: testStagedMetadatas1, 511 } 512 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 513 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 514 Type: encoding.GaugeWithMetadatasType, 515 GaugeWithMetadatas: input, 516 })) 517 dataBuf := enc.Relinquish() 518 defer dataBuf.Close() 519 520 stream := bytes.NewReader(dataBuf.Bytes()) 521 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions().SetMaxMessageSize(1)) 522 require.False(t, it.Next()) 523 require.False(t, it.Next()) 524 } 525 526 func TestUnaggregatedIteratorNextOnClose(t *testing.T) { 527 input := unaggregated.GaugeWithMetadatas{ 528 Gauge: testGauge1, 529 StagedMetadatas: testStagedMetadatas1, 530 } 531 enc := NewUnaggregatedEncoder(NewUnaggregatedOptions()) 532 require.NoError(t, enc.EncodeMessage(encoding.UnaggregatedMessageUnion{ 533 Type: encoding.GaugeWithMetadatasType, 534 GaugeWithMetadatas: input, 535 })) 536 dataBuf := enc.Relinquish() 537 defer dataBuf.Close() 538 539 stream := bytes.NewReader(dataBuf.Bytes()) 540 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 541 require.False(t, it.closed) 542 require.NotNil(t, it.buf) 543 require.Nil(t, it.Err()) 544 545 // Verify that closing the iterator cleans up the state. 546 it.Close() 547 require.False(t, it.Next()) 548 require.False(t, it.Next()) 549 require.True(t, it.closed) 550 require.Nil(t, it.bytesPool) 551 require.Nil(t, it.buf) 552 require.Nil(t, it.Err()) 553 554 // Verify that closing a second time is a no op. 555 it.Close() 556 } 557 558 func TestUnaggregatedIteratorNextOnInvalid(t *testing.T) { 559 buf := make([]byte, 32) 560 binary.PutVarint(buf, 0) 561 stream := bytes.NewReader(buf) 562 563 it := NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 564 require.False(t, it.Next()) 565 require.False(t, it.Next()) 566 567 buf = make([]byte, 32) 568 binary.PutVarint(buf, -1234) 569 stream = bytes.NewReader(buf) 570 it = NewUnaggregatedIterator(stream, NewUnaggregatedOptions()) 571 require.False(t, it.Next()) 572 require.False(t, it.Next()) 573 }