github.com/Jeffail/benthos/v3@v3.65.0/lib/output/elasticsearch.go (about)

     1  package output
     2  
     3  import (
     4  	"github.com/Jeffail/benthos/v3/internal/docs"
     5  	"github.com/Jeffail/benthos/v3/lib/log"
     6  	"github.com/Jeffail/benthos/v3/lib/message/batch"
     7  	"github.com/Jeffail/benthos/v3/lib/metrics"
     8  	"github.com/Jeffail/benthos/v3/lib/output/writer"
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  	sess "github.com/Jeffail/benthos/v3/lib/util/aws/session"
    11  	"github.com/Jeffail/benthos/v3/lib/util/http/auth"
    12  	"github.com/Jeffail/benthos/v3/lib/util/retries"
    13  	"github.com/Jeffail/benthos/v3/lib/util/tls"
    14  )
    15  
    16  //------------------------------------------------------------------------------
    17  
    18  func init() {
    19  	Constructors[TypeElasticsearch] = TypeSpec{
    20  		constructor: fromSimpleConstructor(NewElasticsearch),
    21  		Summary: `
    22  Publishes messages into an Elasticsearch index. If the index does not exist then
    23  it is created with a dynamic mapping.`,
    24  		Description: `
    25  Both the ` + "`id` and `index`" + ` fields can be dynamically set using function
    26  interpolations described [here](/docs/configuration/interpolation#bloblang-queries). When
    27  sending batched messages these interpolations are performed per message part.
    28  
    29  ### AWS
    30  
    31  It's possible to enable AWS connectivity with this output using the ` + "`aws`" + `
    32  fields. However, you may need to set ` + "`sniff` and `healthcheck`" + ` to
    33  false for connections to succeed.`,
    34  		Async:   true,
    35  		Batches: true,
    36  		FieldSpecs: docs.FieldSpecs{
    37  			docs.FieldCommon("urls", "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", []string{"http://localhost:9200"}).Array(),
    38  			docs.FieldCommon("index", "The index to place messages.").IsInterpolated(),
    39  			docs.FieldAdvanced("action", "The action to take on the document.").IsInterpolated().HasOptions("index", "update", "delete"),
    40  			docs.FieldAdvanced("pipeline", "An optional pipeline id to preprocess incoming documents.").IsInterpolated(),
    41  			docs.FieldCommon("id", "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.").IsInterpolated(),
    42  			docs.FieldCommon("type", "The document type."),
    43  			docs.FieldAdvanced("routing", "The routing key to use for the document.").IsInterpolated(),
    44  			docs.FieldAdvanced("sniff", "Prompts Benthos to sniff for brokers to connect to when establishing a connection."),
    45  			docs.FieldAdvanced("healthcheck", "Whether to enable healthchecks."),
    46  			docs.FieldAdvanced("timeout", "The maximum time to wait before abandoning a request (and trying again)."),
    47  			tls.FieldSpec(),
    48  			docs.FieldCommon("max_in_flight", "The maximum number of messages to have in flight at a given time. Increase this to improve throughput."),
    49  		}.Merge(retries.FieldSpecs()).Add(
    50  			auth.BasicAuthFieldSpec(),
    51  			batch.FieldSpec(),
    52  			docs.FieldAdvanced("aws", "Enables and customises connectivity to Amazon Elastic Service.").WithChildren(
    53  				docs.FieldSpecs{
    54  					docs.FieldCommon("enabled", "Whether to connect to Amazon Elastic Service."),
    55  				}.Merge(sess.FieldSpecs())...,
    56  			),
    57  			docs.FieldAdvanced("gzip_compression", "Enable gzip compression on the request side."),
    58  		),
    59  		Categories: []Category{
    60  			CategoryServices,
    61  		},
    62  	}
    63  }
    64  
    65  //------------------------------------------------------------------------------
    66  
    67  // NewElasticsearch creates a new Elasticsearch output type.
    68  func NewElasticsearch(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    69  	elasticWriter, err := writer.NewElasticsearchV2(conf.Elasticsearch, mgr, log, stats)
    70  	if err != nil {
    71  		return nil, err
    72  	}
    73  	var w Type
    74  	if conf.Elasticsearch.MaxInFlight == 1 {
    75  		w, err = NewWriter(
    76  			TypeElasticsearch, elasticWriter, log, stats,
    77  		)
    78  	} else {
    79  		w, err = NewAsyncWriter(
    80  			TypeElasticsearch, conf.Elasticsearch.MaxInFlight, elasticWriter, log, stats,
    81  		)
    82  	}
    83  	if err != nil {
    84  		return w, err
    85  	}
    86  	return NewBatcherFromConfig(conf.Elasticsearch.Batching, w, mgr, log, stats)
    87  }
    88  
    89  //------------------------------------------------------------------------------