github.com/Jeffail/benthos/v3@v3.65.0/lib/test/integration/hdfs_test.go (about)

     1  package integration
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	"github.com/Jeffail/benthos/v3/internal/integration"
     8  	"github.com/colinmarc/hdfs"
     9  	"github.com/ory/dockertest/v3"
    10  	"github.com/ory/dockertest/v3/docker"
    11  	"github.com/stretchr/testify/assert"
    12  	"github.com/stretchr/testify/require"
    13  )
    14  
    15  var _ = registerIntegrationTest("hdfs", func(t *testing.T) {
    16  	t.Skip() // Skip until we fix the static port bindings
    17  	t.Parallel()
    18  
    19  	pool, err := dockertest.NewPool("")
    20  	require.NoError(t, err)
    21  
    22  	pool.MaxWait = time.Second * 30
    23  
    24  	options := &dockertest.RunOptions{
    25  		Repository:   "cybermaggedon/hadoop",
    26  		Tag:          "2.8.2",
    27  		Hostname:     "localhost",
    28  		ExposedPorts: []string{"9000", "50075", "50070", "50010"},
    29  		PortBindings: map[docker.Port][]docker.PortBinding{
    30  			"9000/tcp":  {{HostIP: "", HostPort: "9000"}},
    31  			"50070/tcp": {{HostIP: "", HostPort: "50070"}},
    32  			"50075/tcp": {{HostIP: "", HostPort: "50075"}},
    33  			"50010/tcp": {{HostIP: "", HostPort: "50010"}},
    34  		},
    35  	}
    36  	resource, err := pool.RunWithOptions(options)
    37  	require.NoError(t, err)
    38  	t.Cleanup(func() {
    39  		assert.NoError(t, pool.Purge(resource))
    40  	})
    41  
    42  	resource.Expire(900)
    43  	require.NoError(t, pool.Retry(func() error {
    44  		testFile := "/cluster_ready" + time.Now().Format("20060102150405")
    45  		client, err := hdfs.NewClient(hdfs.ClientOptions{
    46  			Addresses: []string{"localhost:9000"},
    47  			User:      "root",
    48  		})
    49  		if err != nil {
    50  			return err
    51  		}
    52  		fw, err := client.Create(testFile)
    53  		if err != nil {
    54  			return err
    55  		}
    56  		_, err = fw.Write([]byte("testing hdfs reader"))
    57  		if err != nil {
    58  			return err
    59  		}
    60  		err = fw.Close()
    61  		if err != nil {
    62  			return err
    63  		}
    64  		client.Remove(testFile)
    65  		return nil
    66  	}))
    67  
    68  	template := `
    69  output:
    70    hdfs:
    71      hosts: [ localhost:9000 ]
    72      user: root
    73      directory: /$ID
    74      path: ${!count("$ID")}-${!timestamp_unix_nano()}.txt
    75      max_in_flight: $MAX_IN_FLIGHT
    76      batching:
    77        count: $OUTPUT_BATCH_COUNT
    78  
    79  input:
    80    hdfs:
    81      hosts: [ localhost:9000 ]
    82      user: root
    83      directory: /$ID
    84  `
    85  	integration.StreamTests(
    86  		integration.StreamTestOpenCloseIsolated(),
    87  		integration.StreamTestStreamIsolated(10),
    88  		integration.StreamTestSendBatchCountIsolated(10),
    89  	).Run(t, template)
    90  })