github.com/Jeffail/benthos/v3@v3.65.0/lib/test/integration/gcp_pubsub_test.go (about)

     1  package integration
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"testing"
     8  	"time"
     9  
    10  	"cloud.google.com/go/pubsub"
    11  	"github.com/Jeffail/benthos/v3/internal/integration"
    12  	"github.com/ory/dockertest/v3"
    13  	"github.com/stretchr/testify/assert"
    14  	"github.com/stretchr/testify/require"
    15  )
    16  
    17  var _ = registerIntegrationTest("gcp_pubsub", func(t *testing.T) {
    18  	t.Parallel()
    19  
    20  	pool, err := dockertest.NewPool("")
    21  	require.NoError(t, err)
    22  
    23  	pool.MaxWait = time.Second * 30
    24  
    25  	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
    26  		Repository:   "singularities/pubsub-emulator",
    27  		Tag:          "latest",
    28  		ExposedPorts: []string{"8432/tcp"},
    29  		Env: []string{
    30  			"PUBSUB_LISTEN_ADDRESS=0.0.0.0:8432",
    31  			"PUBSUB_PROJECT_ID=benthos-test-project",
    32  		},
    33  	})
    34  	require.NoError(t, err)
    35  	t.Cleanup(func() {
    36  		assert.NoError(t, pool.Purge(resource))
    37  	})
    38  
    39  	require.NoError(t, os.Setenv("PUBSUB_EMULATOR_HOST", fmt.Sprintf("localhost:%v", resource.GetPort("8432/tcp"))))
    40  	require.NotEqual(t, "localhost:", os.Getenv("PUBSUB_EMULATOR_HOST"))
    41  
    42  	resource.Expire(900)
    43  	require.NoError(t, pool.Retry(func() error {
    44  		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    45  		defer cancel()
    46  		client, err := pubsub.NewClient(ctx, "benthos-test-project")
    47  		if err != nil {
    48  			return err
    49  		}
    50  		_, err = client.CreateTopic(ctx, "test-probe-topic-name")
    51  		client.Close()
    52  		return err
    53  	}))
    54  
    55  	template := `
    56  output:
    57    gcp_pubsub:
    58      project: benthos-test-project
    59      topic: topic-$ID
    60      max_in_flight: $MAX_IN_FLIGHT
    61      metadata:
    62        exclude_prefixes: [ $OUTPUT_META_EXCLUDE_PREFIX ]
    63  
    64  input:
    65    gcp_pubsub:
    66      project: benthos-test-project
    67      subscription: sub-$ID
    68  `
    69  	suiteOpts := []integration.StreamTestOptFunc{
    70  		integration.StreamTestOptSleepAfterInput(100 * time.Millisecond),
    71  		integration.StreamTestOptSleepAfterOutput(100 * time.Millisecond),
    72  		integration.StreamTestOptTimeout(time.Minute * 5),
    73  		integration.StreamTestOptPreTest(func(t testing.TB, ctx context.Context, testID string, vars *integration.StreamTestConfigVars) {
    74  			client, err := pubsub.NewClient(ctx, "benthos-test-project")
    75  			require.NoError(t, err)
    76  
    77  			topic, err := client.CreateTopic(ctx, fmt.Sprintf("topic-%v", testID))
    78  			require.NoError(t, err)
    79  
    80  			_, err = client.CreateSubscription(ctx, fmt.Sprintf("sub-%v", testID), pubsub.SubscriptionConfig{
    81  				AckDeadline: time.Second * 10,
    82  				RetryPolicy: &pubsub.RetryPolicy{
    83  					MaximumBackoff: time.Millisecond * 10,
    84  					MinimumBackoff: time.Millisecond * 10,
    85  				},
    86  				Topic: topic,
    87  			})
    88  			require.NoError(t, err)
    89  
    90  			client.Close()
    91  		}),
    92  	}
    93  	suite := integration.StreamTests(
    94  		integration.StreamTestOpenClose(),
    95  		integration.StreamTestMetadata(),
    96  		integration.StreamTestMetadataFilter(),
    97  		integration.StreamTestSendBatches(10, 1000, 10),
    98  		integration.StreamTestStreamSequential(1000),
    99  		integration.StreamTestStreamParallel(1000),
   100  		integration.StreamTestStreamParallelLossy(1000),
   101  		// integration.StreamTestAtLeastOnceDelivery(),
   102  	)
   103  	suite.Run(t, template, suiteOpts...)
   104  	t.Run("with max in flight", func(t *testing.T) {
   105  		t.Parallel()
   106  		suite.Run(
   107  			t, template,
   108  			append([]integration.StreamTestOptFunc{integration.StreamTestOptMaxInFlight(10)}, suiteOpts...)...,
   109  		)
   110  	})
   111  })