github.com/Jeffail/benthos/v3@v3.65.0/lib/output/elasticsearch.go (about) 1 package output 2 3 import ( 4 "github.com/Jeffail/benthos/v3/internal/docs" 5 "github.com/Jeffail/benthos/v3/lib/log" 6 "github.com/Jeffail/benthos/v3/lib/message/batch" 7 "github.com/Jeffail/benthos/v3/lib/metrics" 8 "github.com/Jeffail/benthos/v3/lib/output/writer" 9 "github.com/Jeffail/benthos/v3/lib/types" 10 sess "github.com/Jeffail/benthos/v3/lib/util/aws/session" 11 "github.com/Jeffail/benthos/v3/lib/util/http/auth" 12 "github.com/Jeffail/benthos/v3/lib/util/retries" 13 "github.com/Jeffail/benthos/v3/lib/util/tls" 14 ) 15 16 //------------------------------------------------------------------------------ 17 18 func init() { 19 Constructors[TypeElasticsearch] = TypeSpec{ 20 constructor: fromSimpleConstructor(NewElasticsearch), 21 Summary: ` 22 Publishes messages into an Elasticsearch index. If the index does not exist then 23 it is created with a dynamic mapping.`, 24 Description: ` 25 Both the ` + "`id` and `index`" + ` fields can be dynamically set using function 26 interpolations described [here](/docs/configuration/interpolation#bloblang-queries). When 27 sending batched messages these interpolations are performed per message part. 28 29 ### AWS 30 31 It's possible to enable AWS connectivity with this output using the ` + "`aws`" + ` 32 fields. However, you may need to set ` + "`sniff` and `healthcheck`" + ` to 33 false for connections to succeed.`, 34 Async: true, 35 Batches: true, 36 FieldSpecs: docs.FieldSpecs{ 37 docs.FieldCommon("urls", "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", []string{"http://localhost:9200"}).Array(), 38 docs.FieldCommon("index", "The index to place messages.").IsInterpolated(), 39 docs.FieldAdvanced("action", "The action to take on the document.").IsInterpolated().HasOptions("index", "update", "delete"), 40 docs.FieldAdvanced("pipeline", "An optional pipeline id to preprocess incoming documents.").IsInterpolated(), 41 docs.FieldCommon("id", "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.").IsInterpolated(), 42 docs.FieldCommon("type", "The document type."), 43 docs.FieldAdvanced("routing", "The routing key to use for the document.").IsInterpolated(), 44 docs.FieldAdvanced("sniff", "Prompts Benthos to sniff for brokers to connect to when establishing a connection."), 45 docs.FieldAdvanced("healthcheck", "Whether to enable healthchecks."), 46 docs.FieldAdvanced("timeout", "The maximum time to wait before abandoning a request (and trying again)."), 47 tls.FieldSpec(), 48 docs.FieldCommon("max_in_flight", "The maximum number of messages to have in flight at a given time. Increase this to improve throughput."), 49 }.Merge(retries.FieldSpecs()).Add( 50 auth.BasicAuthFieldSpec(), 51 batch.FieldSpec(), 52 docs.FieldAdvanced("aws", "Enables and customises connectivity to Amazon Elastic Service.").WithChildren( 53 docs.FieldSpecs{ 54 docs.FieldCommon("enabled", "Whether to connect to Amazon Elastic Service."), 55 }.Merge(sess.FieldSpecs())..., 56 ), 57 docs.FieldAdvanced("gzip_compression", "Enable gzip compression on the request side."), 58 ), 59 Categories: []Category{ 60 CategoryServices, 61 }, 62 } 63 } 64 65 //------------------------------------------------------------------------------ 66 67 // NewElasticsearch creates a new Elasticsearch output type. 68 func NewElasticsearch(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) { 69 elasticWriter, err := writer.NewElasticsearchV2(conf.Elasticsearch, mgr, log, stats) 70 if err != nil { 71 return nil, err 72 } 73 var w Type 74 if conf.Elasticsearch.MaxInFlight == 1 { 75 w, err = NewWriter( 76 TypeElasticsearch, elasticWriter, log, stats, 77 ) 78 } else { 79 w, err = NewAsyncWriter( 80 TypeElasticsearch, conf.Elasticsearch.MaxInFlight, elasticWriter, log, stats, 81 ) 82 } 83 if err != nil { 84 return w, err 85 } 86 return NewBatcherFromConfig(conf.Elasticsearch.Batching, w, mgr, log, stats) 87 } 88 89 //------------------------------------------------------------------------------