github.com/Jeffail/benthos/v3@v3.65.0/lib/test/integration/elasticsearch_test.go (about)

     1  package integration
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"net/http"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/Jeffail/benthos/v3/internal/integration"
    11  	"github.com/olivere/elastic/v7"
    12  	"github.com/ory/dockertest/v3"
    13  	"github.com/stretchr/testify/assert"
    14  	"github.com/stretchr/testify/require"
    15  )
    16  
    17  var elasticIndex = `{
    18  	"settings":{
    19  		"number_of_shards": 1,
    20  		"number_of_replicas": 0
    21  	},
    22  	"mappings":{
    23  		"properties": {
    24  			"user":{
    25  				"type":"keyword"
    26  			},
    27  			"message":{
    28  				"type": "text",
    29  				"store": true,
    30  				"fielddata": true
    31  			}
    32  		}
    33  	}
    34  }`
    35  
    36  var _ = registerIntegrationTest("elasticsearch", func(t *testing.T) {
    37  	t.Skip("This uses a ton of memory so we don't run it by default")
    38  
    39  	t.Parallel()
    40  
    41  	pool, err := dockertest.NewPool("")
    42  	require.NoError(t, err)
    43  
    44  	pool.MaxWait = time.Second * 30
    45  	resource, err := pool.Run("elasticsearch", "7.17.0", []string{
    46  		"discovery.type=single-node",
    47  	})
    48  	require.NoError(t, err)
    49  	t.Cleanup(func() {
    50  		assert.NoError(t, pool.Purge(resource))
    51  	})
    52  
    53  	var client *elastic.Client
    54  	if err = pool.Retry(func() error {
    55  		opts := []elastic.ClientOptionFunc{
    56  			elastic.SetURL(fmt.Sprintf("http://localhost:%v", resource.GetPort("9200/tcp"))),
    57  			elastic.SetHttpClient(&http.Client{
    58  				Timeout: time.Second,
    59  			}),
    60  			elastic.SetSniff(false),
    61  		}
    62  
    63  		var cerr error
    64  		if client, cerr = elastic.NewClient(opts...); cerr == nil {
    65  			_, cerr = client.
    66  				CreateIndex("test_conn_index").
    67  				Timeout("20s").
    68  				Body(elasticIndex).
    69  				Do(context.Background())
    70  		}
    71  		return cerr
    72  	}); err != nil {
    73  		t.Fatalf("Could not connect to docker resource: %s", err)
    74  	}
    75  
    76  	resource.Expire(900)
    77  
    78  	template := `
    79  output:
    80    elasticsearch:
    81      urls:
    82        - http://localhost:$PORT
    83      index: $ID
    84      id: ${!json("id")}
    85      type: doc
    86      sniff: false
    87  `
    88  	queryGetFn := func(ctx context.Context, testID, messageID string) (string, []string, error) {
    89  		res, err := client.Get().
    90  			Index(testID).
    91  			Id(messageID).
    92  			Do(ctx)
    93  		if err != nil {
    94  			return "", nil, err
    95  		}
    96  
    97  		if !res.Found {
    98  			return "", nil, fmt.Errorf("document %v not found", messageID)
    99  		}
   100  
   101  		resBytes, err := res.Source.MarshalJSON()
   102  		if err != nil {
   103  			return "", nil, err
   104  		}
   105  		return string(resBytes), nil, nil
   106  	}
   107  
   108  	suite := integration.StreamTests(
   109  		integration.StreamTestOutputOnlySendSequential(10, queryGetFn),
   110  		integration.StreamTestOutputOnlySendBatch(10, queryGetFn),
   111  	)
   112  	suite.Run(
   113  		t, template,
   114  		integration.StreamTestOptPort(resource.GetPort("9200/tcp")),
   115  	)
   116  })
   117  
   118  var _ = registerIntegrationBench("elasticsearch", func(b *testing.B) {
   119  	pool, err := dockertest.NewPool("")
   120  	require.NoError(b, err)
   121  
   122  	pool.MaxWait = time.Second * 30
   123  	resource, err := pool.Run("elasticsearch", "7.13.4", []string{
   124  		"discovery.type=single-node",
   125  	})
   126  	require.NoError(b, err)
   127  	b.Cleanup(func() {
   128  		assert.NoError(b, pool.Purge(resource))
   129  	})
   130  
   131  	var client *elastic.Client
   132  	if err = pool.Retry(func() error {
   133  		opts := []elastic.ClientOptionFunc{
   134  			elastic.SetURL(fmt.Sprintf("http://localhost:%v", resource.GetPort("9200/tcp"))),
   135  			elastic.SetHttpClient(&http.Client{
   136  				Timeout: time.Second,
   137  			}),
   138  			elastic.SetSniff(false),
   139  		}
   140  
   141  		var cerr error
   142  		if client, cerr = elastic.NewClient(opts...); cerr == nil {
   143  			_, cerr = client.
   144  				CreateIndex("test_conn_index").
   145  				Timeout("20s").
   146  				Body(elasticIndex).
   147  				Do(context.Background())
   148  		}
   149  		return cerr
   150  	}); err != nil {
   151  		b.Fatalf("Could not connect to docker resource: %s", err)
   152  	}
   153  
   154  	resource.Expire(900)
   155  
   156  	template := `
   157  output:
   158    elasticsearch:
   159      urls:
   160        - http://localhost:$PORT
   161      index: $ID
   162      id: ${!json("id")}
   163      type: doc
   164      sniff: false
   165  `
   166  	suite := integration.StreamBenchs(
   167  		integration.StreamBenchWrite(20),
   168  		integration.StreamBenchWrite(10),
   169  		integration.StreamBenchWrite(1),
   170  	)
   171  	suite.Run(
   172  		b, template,
   173  		integration.StreamTestOptPort(resource.GetPort("9200/tcp")),
   174  	)
   175  })