github.com/Jeffail/benthos/v3@v3.65.0/internal/integration/stream_test_definitions.go (about) 1 package integration 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "strconv" 8 "sync" 9 "testing" 10 "time" 11 12 "github.com/Jeffail/benthos/v3/lib/message" 13 "github.com/Jeffail/benthos/v3/lib/response" 14 "github.com/Jeffail/benthos/v3/lib/types" 15 16 "github.com/stretchr/testify/assert" 17 "github.com/stretchr/testify/require" 18 ) 19 20 // StreamTestOpenClose ensures that both the input and output can be started and 21 // stopped within a reasonable length of time. A single message is sent to check 22 // the connection. 23 func StreamTestOpenClose() StreamTestDefinition { 24 return namedStreamTest( 25 "can open and close", 26 func(t *testing.T, env *streamTestEnvironment) { 27 t.Parallel() 28 29 tranChan := make(chan types.Transaction) 30 input, output := initConnectors(t, tranChan, env) 31 t.Cleanup(func() { 32 closeConnectors(t, input, output) 33 }) 34 35 require.NoError(t, sendMessage(env.ctx, t, tranChan, "hello world")) 36 messageMatch(t, receiveMessage(env.ctx, t, input.TransactionChan(), nil), "hello world") 37 }, 38 ) 39 } 40 41 // StreamTestOpenCloseIsolated ensures that both the input and output can be 42 // started and stopped within a reasonable length of time. A single message is 43 // sent to check the connection but the input is only started after the message 44 // is sent. 45 func StreamTestOpenCloseIsolated() StreamTestDefinition { 46 return namedStreamTest( 47 "can open and close isolated", 48 func(t *testing.T, env *streamTestEnvironment) { 49 t.Parallel() 50 51 tranChan := make(chan types.Transaction) 52 output := initOutput(t, tranChan, env) 53 t.Cleanup(func() { 54 closeConnectors(t, nil, output) 55 }) 56 require.NoError(t, sendMessage(env.ctx, t, tranChan, "hello world")) 57 58 input := initInput(t, env) 59 t.Cleanup(func() { 60 closeConnectors(t, input, nil) 61 }) 62 messageMatch(t, receiveMessage(env.ctx, t, input.TransactionChan(), nil), "hello world") 63 }, 64 ) 65 } 66 67 // StreamTestMetadata ensures that we are able to send and receive metadata 68 // values. 69 func StreamTestMetadata() StreamTestDefinition { 70 return namedStreamTest( 71 "can send and receive metadata", 72 func(t *testing.T, env *streamTestEnvironment) { 73 t.Parallel() 74 75 tranChan := make(chan types.Transaction) 76 input, output := initConnectors(t, tranChan, env) 77 t.Cleanup(func() { 78 closeConnectors(t, input, output) 79 }) 80 81 require.NoError(t, sendMessage( 82 env.ctx, t, tranChan, 83 "hello world", 84 "foo", "foo_value", 85 "bar", "bar_value", 86 )) 87 messageMatch( 88 t, receiveMessage(env.ctx, t, input.TransactionChan(), nil), 89 "hello world", 90 "foo", "foo_value", 91 "bar", "bar_value", 92 ) 93 }, 94 ) 95 } 96 97 // StreamTestMetadataFilter ensures that we are able to send and receive 98 // metadata values, and that they are filtered. The provided config template 99 // should inject the variable $OUTPUT_META_EXCLUDE_PREFIX into the output 100 // metadata filter field. 101 func StreamTestMetadataFilter() StreamTestDefinition { 102 return namedStreamTest( 103 "can send and receive metadata filtered", 104 func(t *testing.T, env *streamTestEnvironment) { 105 t.Parallel() 106 107 env.configVars.OutputMetaExcludePrefix = "f" 108 109 tranChan := make(chan types.Transaction) 110 input, output := initConnectors(t, tranChan, env) 111 t.Cleanup(func() { 112 closeConnectors(t, input, output) 113 }) 114 115 require.NoError(t, sendMessage( 116 env.ctx, t, tranChan, 117 "hello world", 118 "foo", "foo_value", 119 "bar", "bar_value", 120 )) 121 122 p := receiveMessage(env.ctx, t, input.TransactionChan(), nil) 123 assert.Empty(t, p.Metadata().Get("foo")) 124 messageMatch(t, p, "hello world", "bar", "bar_value") 125 }, 126 ) 127 } 128 129 // StreamTestSendBatch ensures we can send a batch of a given size. 130 func StreamTestSendBatch(n int) StreamTestDefinition { 131 return namedStreamTest( 132 "can send a message batch", 133 func(t *testing.T, env *streamTestEnvironment) { 134 t.Parallel() 135 136 tranChan := make(chan types.Transaction) 137 input, output := initConnectors(t, tranChan, env) 138 t.Cleanup(func() { 139 closeConnectors(t, input, output) 140 }) 141 142 set := map[string][]string{} 143 payloads := []string{} 144 for i := 0; i < n; i++ { 145 payload := fmt.Sprintf("hello world %v", i) 146 set[payload] = nil 147 payloads = append(payloads, payload) 148 } 149 err := sendBatch(env.ctx, t, tranChan, payloads) 150 assert.NoError(t, err) 151 152 for len(set) > 0 { 153 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 154 } 155 }, 156 ) 157 } 158 159 // StreamTestSendBatches ensures that we can send N batches of M parallelism. 160 func StreamTestSendBatches(batchSize, batches, parallelism int) StreamTestDefinition { 161 return namedStreamTest( 162 "can send many message batches", 163 func(t *testing.T, env *streamTestEnvironment) { 164 t.Parallel() 165 166 require.Greater(t, parallelism, 0) 167 168 tranChan := make(chan types.Transaction) 169 input, output := initConnectors(t, tranChan, env) 170 t.Cleanup(func() { 171 closeConnectors(t, input, output) 172 }) 173 174 set := map[string][]string{} 175 for j := 0; j < batches; j++ { 176 for i := 0; i < batchSize; i++ { 177 payload := fmt.Sprintf("hello world %v", j*batches+i) 178 set[payload] = nil 179 } 180 } 181 182 batchChan := make(chan []string) 183 184 var wg sync.WaitGroup 185 for k := 0; k < parallelism; k++ { 186 wg.Add(1) 187 go func() { 188 defer wg.Done() 189 for { 190 batch, open := <-batchChan 191 if !open { 192 return 193 } 194 assert.NoError(t, sendBatch(env.ctx, t, tranChan, batch)) 195 } 196 }() 197 } 198 199 wg.Add(1) 200 go func() { 201 defer wg.Done() 202 for len(set) > 0 { 203 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 204 } 205 }() 206 207 for j := 0; j < batches; j++ { 208 payloads := []string{} 209 for i := 0; i < batchSize; i++ { 210 payload := fmt.Sprintf("hello world %v", j*batches+i) 211 payloads = append(payloads, payload) 212 } 213 batchChan <- payloads 214 } 215 close(batchChan) 216 217 wg.Wait() 218 }, 219 ) 220 } 221 222 // StreamTestSendBatchCount ensures we can send batches using a configured batch 223 // count. 224 func StreamTestSendBatchCount(n int) StreamTestDefinition { 225 return namedStreamTest( 226 "can send messages with an output batch count", 227 func(t *testing.T, env *streamTestEnvironment) { 228 t.Parallel() 229 230 env.configVars.OutputBatchCount = n 231 232 tranChan := make(chan types.Transaction) 233 input, output := initConnectors(t, tranChan, env) 234 t.Cleanup(func() { 235 closeConnectors(t, input, output) 236 }) 237 238 resChan := make(chan types.Response) 239 240 set := map[string][]string{} 241 for i := 0; i < n; i++ { 242 payload := fmt.Sprintf("hello world %v", i) 243 set[payload] = nil 244 msg := message.New(nil) 245 msg.Append(message.NewPart([]byte(payload))) 246 select { 247 case tranChan <- types.NewTransaction(msg, resChan): 248 case res := <-resChan: 249 t.Fatalf("premature response: %v", res.Error()) 250 case <-env.ctx.Done(): 251 t.Fatal("timed out on send") 252 } 253 } 254 255 for i := 0; i < n; i++ { 256 select { 257 case res := <-resChan: 258 assert.NoError(t, res.Error()) 259 case <-env.ctx.Done(): 260 t.Fatal("timed out on response") 261 } 262 } 263 264 for len(set) > 0 { 265 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 266 } 267 }, 268 ) 269 } 270 271 // StreamTestSendBatchCountIsolated checks batches can be sent and then 272 // received. The input is created after the output has written data. 273 func StreamTestSendBatchCountIsolated(n int) StreamTestDefinition { 274 return namedStreamTest( 275 "can send messages with an output batch count isolated", 276 func(t *testing.T, env *streamTestEnvironment) { 277 t.Parallel() 278 279 env.configVars.OutputBatchCount = n 280 281 tranChan := make(chan types.Transaction) 282 output := initOutput(t, tranChan, env) 283 t.Cleanup(func() { 284 closeConnectors(t, nil, output) 285 }) 286 287 resChan := make(chan types.Response) 288 289 set := map[string][]string{} 290 for i := 0; i < n; i++ { 291 payload := fmt.Sprintf("hello world %v", i) 292 set[payload] = nil 293 msg := message.New(nil) 294 msg.Append(message.NewPart([]byte(payload))) 295 select { 296 case tranChan <- types.NewTransaction(msg, resChan): 297 case res := <-resChan: 298 t.Fatalf("premature response: %v", res.Error()) 299 case <-env.ctx.Done(): 300 t.Fatal("timed out on send") 301 } 302 } 303 304 for i := 0; i < n; i++ { 305 select { 306 case res := <-resChan: 307 assert.NoError(t, res.Error()) 308 case <-env.ctx.Done(): 309 t.Fatal("timed out on response") 310 } 311 } 312 313 input := initInput(t, env) 314 t.Cleanup(func() { 315 closeConnectors(t, input, nil) 316 }) 317 318 for len(set) > 0 { 319 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 320 } 321 }, 322 ) 323 } 324 325 // StreamTestReceiveBatchCount tests that batches can be consumed with an input 326 // configured batch count. 327 func StreamTestReceiveBatchCount(n int) StreamTestDefinition { 328 return namedStreamTest( 329 "can send messages with an input batch count", 330 func(t *testing.T, env *streamTestEnvironment) { 331 t.Parallel() 332 333 env.configVars.InputBatchCount = n 334 335 tranChan := make(chan types.Transaction) 336 input, output := initConnectors(t, tranChan, env) 337 t.Cleanup(func() { 338 closeConnectors(t, input, output) 339 }) 340 341 set := map[string][]string{} 342 343 for i := 0; i < n; i++ { 344 payload := fmt.Sprintf("hello world: %v", i) 345 set[payload] = nil 346 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 347 } 348 349 var tran types.Transaction 350 select { 351 case tran = <-input.TransactionChan(): 352 case <-env.ctx.Done(): 353 t.Fatal("timed out on receive") 354 } 355 356 assert.Equal(t, n, tran.Payload.Len()) 357 tran.Payload.Iter(func(_ int, p types.Part) error { 358 messageInSet(t, true, false, p, set) 359 return nil 360 }) 361 362 select { 363 case tran.ResponseChan <- response.NewAck(): 364 case <-env.ctx.Done(): 365 t.Fatal("timed out on response") 366 } 367 }, 368 ) 369 } 370 371 // StreamTestStreamSequential tests that data can be sent and received, where 372 // data is sent sequentially. 373 func StreamTestStreamSequential(n int) StreamTestDefinition { 374 return namedStreamTest( 375 "can send and receive data sequentially", 376 func(t *testing.T, env *streamTestEnvironment) { 377 t.Parallel() 378 379 tranChan := make(chan types.Transaction) 380 input, output := initConnectors(t, tranChan, env) 381 t.Cleanup(func() { 382 closeConnectors(t, input, output) 383 }) 384 385 set := map[string][]string{} 386 387 for i := 0; i < n; i++ { 388 payload := fmt.Sprintf("hello world: %v", i) 389 set[payload] = nil 390 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 391 } 392 393 for len(set) > 0 { 394 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 395 } 396 }, 397 ) 398 } 399 400 // StreamTestStreamIsolated tests that data can be sent and received, where data 401 // is sent sequentially. The input is created after the output has written data. 402 func StreamTestStreamIsolated(n int) StreamTestDefinition { 403 return namedStreamTest( 404 "can send and receive data isolated", 405 func(t *testing.T, env *streamTestEnvironment) { 406 t.Parallel() 407 408 tranChan := make(chan types.Transaction) 409 output := initOutput(t, tranChan, env) 410 t.Cleanup(func() { 411 closeConnectors(t, nil, output) 412 }) 413 414 set := map[string][]string{} 415 416 for i := 0; i < n; i++ { 417 payload := fmt.Sprintf("hello world: %v", i) 418 set[payload] = nil 419 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 420 } 421 422 input := initInput(t, env) 423 t.Cleanup(func() { 424 closeConnectors(t, input, nil) 425 }) 426 427 for len(set) > 0 { 428 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 429 } 430 }, 431 ) 432 } 433 434 // StreamTestCheckpointCapture ensures that data received out of order doesn't 435 // result in wrongly acknowledged messages. 436 func StreamTestCheckpointCapture() StreamTestDefinition { 437 return namedStreamTest( 438 "respects checkpointed offsets", 439 func(t *testing.T, env *streamTestEnvironment) { 440 t.Parallel() 441 442 tranChan := make(chan types.Transaction) 443 input, output := initConnectors(t, tranChan, env) 444 t.Cleanup(func() { 445 closeConnectors(t, nil, output) 446 }) 447 448 go func() { 449 require.NoError(t, sendMessage(env.ctx, t, tranChan, "A")) 450 require.NoError(t, sendMessage(env.ctx, t, tranChan, "B")) 451 require.NoError(t, sendMessage(env.ctx, t, tranChan, "C")) 452 require.NoError(t, sendMessage(env.ctx, t, tranChan, "D")) 453 require.NoError(t, sendMessage(env.ctx, t, tranChan, "E")) 454 }() 455 456 var msg types.Part 457 responseChans := make([]chan<- types.Response, 5) 458 459 msg, responseChans[0] = receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 460 assert.Equal(t, "A", string(msg.Get())) 461 sendResponse(env.ctx, t, responseChans[0], nil) 462 463 msg, responseChans[1] = receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 464 assert.Equal(t, "B", string(msg.Get())) 465 sendResponse(env.ctx, t, responseChans[1], nil) 466 467 msg, responseChans[2] = receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 468 assert.Equal(t, "C", string(msg.Get())) 469 470 msg, responseChans[3] = receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 471 assert.Equal(t, "D", string(msg.Get())) 472 sendResponse(env.ctx, t, responseChans[3], nil) 473 474 msg, responseChans[4] = receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 475 assert.Equal(t, "E", string(msg.Get())) 476 477 sendResponse(env.ctx, t, responseChans[2], errors.New("rejecting just cus")) 478 sendResponse(env.ctx, t, responseChans[4], errors.New("rejecting just cus")) 479 480 closeConnectors(t, input, nil) 481 482 select { 483 case <-time.After(time.Second * 5): 484 case <-env.ctx.Done(): 485 t.Fatal(env.ctx.Err()) 486 } 487 488 input = initInput(t, env) 489 t.Cleanup(func() { 490 closeConnectors(t, input, nil) 491 }) 492 493 msg = receiveMessage(env.ctx, t, input.TransactionChan(), nil) 494 assert.Equal(t, "C", string(msg.Get())) 495 496 msg = receiveMessage(env.ctx, t, input.TransactionChan(), nil) 497 assert.Equal(t, "D", string(msg.Get())) 498 499 msg = receiveMessage(env.ctx, t, input.TransactionChan(), nil) 500 assert.Equal(t, "E", string(msg.Get())) 501 }, 502 ) 503 } 504 505 // StreamTestStreamParallel tests data transfer with parallel senders. 506 func StreamTestStreamParallel(n int) StreamTestDefinition { 507 return namedStreamTest( 508 "can send and receive data in parallel", 509 func(t *testing.T, env *streamTestEnvironment) { 510 t.Parallel() 511 512 tranChan := make(chan types.Transaction, n) 513 input, output := initConnectors(t, tranChan, env) 514 t.Cleanup(func() { 515 closeConnectors(t, input, output) 516 }) 517 518 set := map[string][]string{} 519 for i := 0; i < n; i++ { 520 payload := fmt.Sprintf("hello world: %v", i) 521 set[payload] = nil 522 } 523 524 wg := sync.WaitGroup{} 525 wg.Add(2) 526 527 go func() { 528 defer wg.Done() 529 for i := 0; i < n; i++ { 530 payload := fmt.Sprintf("hello world: %v", i) 531 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 532 } 533 }() 534 535 go func() { 536 defer wg.Done() 537 for len(set) > 0 { 538 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 539 } 540 }() 541 542 wg.Wait() 543 }, 544 ) 545 } 546 547 // StreamTestStreamSaturatedUnacked writes N messages as a backlog, then 548 // consumes half of those messages without acking them, and then after a pause 549 // acknowledges them all and resumes consuming. 550 // 551 // The purpose of this test is to ensure that after a period of back pressure is 552 // applied the input correctly resumes. 553 func StreamTestStreamSaturatedUnacked(n int) StreamTestDefinition { 554 return namedStreamTest( 555 "can consume data without acking and resume", 556 func(t *testing.T, env *streamTestEnvironment) { 557 t.Parallel() 558 559 tranChan := make(chan types.Transaction, n) 560 input, output := initConnectors(t, tranChan, env) 561 t.Cleanup(func() { 562 closeConnectors(t, input, output) 563 }) 564 565 set := map[string][]string{} 566 for i := 0; i < n*2; i++ { 567 payload := fmt.Sprintf("hello world: %v", i) 568 set[payload] = nil 569 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 570 } 571 572 resChans := make([]chan<- types.Response, n/2) 573 for i := range resChans { 574 var b types.Part 575 b, resChans[i] = receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 576 messageInSet(t, true, env.allowDuplicateMessages, b, set) 577 } 578 579 <-time.After(time.Second * 5) 580 for _, rChan := range resChans { 581 sendResponse(env.ctx, t, rChan, nil) 582 } 583 584 // Consume all remaining messages 585 for len(set) > 0 { 586 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 587 } 588 }, 589 ) 590 } 591 592 // StreamTestAtLeastOnceDelivery ensures data is delivered through nacks and 593 // restarts. 594 func StreamTestAtLeastOnceDelivery() StreamTestDefinition { 595 return namedStreamTest( 596 "at least once delivery", 597 func(t *testing.T, env *streamTestEnvironment) { 598 t.Parallel() 599 600 tranChan := make(chan types.Transaction) 601 input, output := initConnectors(t, tranChan, env) 602 t.Cleanup(func() { 603 closeConnectors(t, nil, output) 604 }) 605 606 expectedMessages := map[string]struct{}{ 607 "A": {}, "B": {}, "C": {}, "D": {}, "E": {}, 608 } 609 go func() { 610 for k := range expectedMessages { 611 require.NoError(t, sendMessage(env.ctx, t, tranChan, k)) 612 } 613 }() 614 615 var msg types.Part 616 badResponseChans := []chan<- types.Response{} 617 618 for i := 0; i < len(expectedMessages); i++ { 619 msg, responseChan := receiveMessageNoRes(env.ctx, t, input.TransactionChan()) 620 key := string(msg.Get()) 621 assert.Contains(t, expectedMessages, key) 622 delete(expectedMessages, key) 623 if key != "C" && key != "E" { 624 sendResponse(env.ctx, t, responseChan, nil) 625 } else { 626 badResponseChans = append(badResponseChans, responseChan) 627 } 628 } 629 630 for _, rChan := range badResponseChans { 631 sendResponse(env.ctx, t, rChan, errors.New("rejecting just cus")) 632 } 633 634 select { 635 case <-time.After(time.Second * 5): 636 case <-env.ctx.Done(): 637 t.Fatal(env.ctx.Err()) 638 } 639 640 closeConnectors(t, input, nil) 641 642 select { 643 case <-time.After(time.Second * 5): 644 case <-env.ctx.Done(): 645 t.Fatal(env.ctx.Err()) 646 } 647 648 input = initInput(t, env) 649 t.Cleanup(func() { 650 closeConnectors(t, input, nil) 651 }) 652 653 expectedMessages = map[string]struct{}{ 654 "C": {}, "E": {}, 655 } 656 657 for i := 0; i < len(expectedMessages); i++ { 658 msg = receiveMessage(env.ctx, t, input.TransactionChan(), nil) 659 key := string(msg.Get()) 660 assert.Contains(t, expectedMessages, key) 661 delete(expectedMessages, key) 662 } 663 }, 664 ) 665 } 666 667 // StreamTestStreamParallelLossy ensures that data is delivered through parallel 668 // nacks. 669 func StreamTestStreamParallelLossy(n int) StreamTestDefinition { 670 return namedStreamTest( 671 "can send and receive data in parallel lossy", 672 func(t *testing.T, env *streamTestEnvironment) { 673 t.Parallel() 674 675 tranChan := make(chan types.Transaction) 676 input, output := initConnectors(t, tranChan, env) 677 t.Cleanup(func() { 678 closeConnectors(t, input, output) 679 }) 680 681 set := map[string][]string{} 682 for i := 0; i < n; i++ { 683 payload := fmt.Sprintf("hello world: %v", i) 684 set[payload] = nil 685 } 686 687 wg := sync.WaitGroup{} 688 wg.Add(2) 689 690 go func() { 691 defer wg.Done() 692 for i := 0; i < n; i++ { 693 payload := fmt.Sprintf("hello world: %v", i) 694 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 695 } 696 }() 697 698 go func() { 699 defer wg.Done() 700 rejected := 0 701 for i := 0; i < n; i++ { 702 if i%10 == 1 { 703 rejected++ 704 messageInSet( 705 t, false, true, 706 receiveMessage(env.ctx, t, input.TransactionChan(), errors.New("rejected just cus")), 707 set, 708 ) 709 } else { 710 messageInSet(t, true, true, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 711 } 712 } 713 714 t.Log("Finished first loop, looping through rejected messages.") 715 for len(set) > 0 { 716 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 717 } 718 }() 719 720 wg.Wait() 721 }, 722 ) 723 } 724 725 // StreamTestStreamParallelLossyThroughReconnect ensures data is delivered 726 // through nacks and restarts. 727 func StreamTestStreamParallelLossyThroughReconnect(n int) StreamTestDefinition { 728 return namedStreamTest( 729 "can send and receive data in parallel lossy through reconnect", 730 func(t *testing.T, env *streamTestEnvironment) { 731 t.Parallel() 732 733 tranChan := make(chan types.Transaction) 734 input, output := initConnectors(t, tranChan, env) 735 t.Cleanup(func() { 736 closeConnectors(t, nil, output) 737 }) 738 739 set := map[string][]string{} 740 for i := 0; i < n; i++ { 741 payload := fmt.Sprintf("hello world: %v", i) 742 set[payload] = nil 743 } 744 745 wg := sync.WaitGroup{} 746 wg.Add(2) 747 748 go func() { 749 defer wg.Done() 750 for i := 0; i < n; i++ { 751 payload := fmt.Sprintf("hello world: %v", i) 752 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload)) 753 } 754 }() 755 756 go func() { 757 defer wg.Done() 758 rejected := 0 759 for i := 0; i < n; i++ { 760 if i%10 == 1 { 761 rejected++ 762 messageInSet( 763 t, false, env.allowDuplicateMessages, 764 receiveMessage(env.ctx, t, input.TransactionChan(), errors.New("rejected just cus")), 765 set, 766 ) 767 } else { 768 messageInSet(t, true, env.allowDuplicateMessages, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 769 } 770 } 771 772 closeConnectors(t, input, nil) 773 774 input = initInput(t, env) 775 t.Cleanup(func() { 776 closeConnectors(t, input, nil) 777 }) 778 779 t.Log("Finished first loop, looping through rejected messages.") 780 for len(set) > 0 { 781 messageInSet(t, true, true, receiveMessage(env.ctx, t, input.TransactionChan(), nil), set) 782 } 783 }() 784 785 wg.Wait() 786 }, 787 ) 788 } 789 790 // GetMessageFunc is a closure used to extract message contents from an output 791 // directly and can be used to test outputs without the need for an input in the 792 // config template. 793 type GetMessageFunc func(ctx context.Context, testID, messageID string) (string, []string, error) 794 795 // StreamTestOutputOnlySendSequential tests a config template without an input. 796 func StreamTestOutputOnlySendSequential(n int, getFn GetMessageFunc) StreamTestDefinition { 797 return namedStreamTest( 798 "can send to output", 799 func(t *testing.T, env *streamTestEnvironment) { 800 t.Parallel() 801 802 tranChan := make(chan types.Transaction) 803 output := initOutput(t, tranChan, env) 804 t.Cleanup(func() { 805 closeConnectors(t, nil, output) 806 }) 807 808 set := map[string]string{} 809 for i := 0; i < n; i++ { 810 id := strconv.Itoa(i) 811 payload := fmt.Sprintf(`{"content":"hello world","id":%v}`, id) 812 set[id] = payload 813 require.NoError(t, sendMessage(env.ctx, t, tranChan, payload, "id", id)) 814 } 815 816 for k, exp := range set { 817 act, _, err := getFn(env.ctx, env.configVars.ID, k) 818 require.NoError(t, err) 819 assert.Equal(t, exp, act) 820 } 821 }, 822 ) 823 } 824 825 // StreamTestOutputOnlySendBatch tests a config template without an input. 826 func StreamTestOutputOnlySendBatch(n int, getFn GetMessageFunc) StreamTestDefinition { 827 return namedStreamTest( 828 "can send to output as batch", 829 func(t *testing.T, env *streamTestEnvironment) { 830 t.Parallel() 831 832 tranChan := make(chan types.Transaction) 833 output := initOutput(t, tranChan, env) 834 t.Cleanup(func() { 835 closeConnectors(t, nil, output) 836 }) 837 838 set := map[string]string{} 839 batch := []string{} 840 for i := 0; i < n; i++ { 841 id := strconv.Itoa(i) 842 payload := fmt.Sprintf(`{"content":"hello world","id":%v}`, id) 843 set[id] = payload 844 batch = append(batch, payload) 845 } 846 require.NoError(t, sendBatch(env.ctx, t, tranChan, batch)) 847 848 for k, exp := range set { 849 act, _, err := getFn(env.ctx, env.configVars.ID, k) 850 require.NoError(t, err) 851 assert.Equal(t, exp, act) 852 } 853 }, 854 ) 855 } 856 857 // StreamTestOutputOnlyOverride tests a config template without an input where 858 // duplicate IDs are sent (where we expect updates). 859 func StreamTestOutputOnlyOverride(getFn GetMessageFunc) StreamTestDefinition { 860 return namedStreamTest( 861 "can send to output and override value", 862 func(t *testing.T, env *streamTestEnvironment) { 863 t.Parallel() 864 865 tranChan := make(chan types.Transaction) 866 output := initOutput(t, tranChan, env) 867 t.Cleanup(func() { 868 closeConnectors(t, nil, output) 869 }) 870 871 first := `{"content":"this should be overridden","id":1}` 872 exp := `{"content":"hello world","id":1}` 873 require.NoError(t, sendMessage(env.ctx, t, tranChan, first)) 874 require.NoError(t, sendMessage(env.ctx, t, tranChan, exp)) 875 876 act, _, err := getFn(env.ctx, env.configVars.ID, "1") 877 require.NoError(t, err) 878 assert.Equal(t, exp, act) 879 }, 880 ) 881 }