github.com/Jeffail/benthos/v3@v3.65.0/lib/output/aws_kinesis.go (about)

     1  package output
     2  
     3  import (
     4  	"github.com/Jeffail/benthos/v3/internal/docs"
     5  	"github.com/Jeffail/benthos/v3/lib/log"
     6  	"github.com/Jeffail/benthos/v3/lib/message/batch"
     7  	"github.com/Jeffail/benthos/v3/lib/metrics"
     8  	"github.com/Jeffail/benthos/v3/lib/output/writer"
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  	"github.com/Jeffail/benthos/v3/lib/util/aws/session"
    11  	"github.com/Jeffail/benthos/v3/lib/util/retries"
    12  )
    13  
    14  //------------------------------------------------------------------------------
    15  
    16  func init() {
    17  	Constructors[TypeAWSKinesis] = TypeSpec{
    18  		constructor: fromSimpleConstructor(NewAWSKinesis),
    19  		Version:     "3.36.0",
    20  		Summary: `
    21  Sends messages to a Kinesis stream.`,
    22  		Description: `
    23  Both the ` + "`partition_key`" + `(required) and ` + "`hash_key`" + ` (optional)
    24  fields can be dynamically set using function interpolations described
    25  [here](/docs/configuration/interpolation#bloblang-queries). When sending batched messages the
    26  interpolations are performed per message part.
    27  
    28  ### Credentials
    29  
    30  By default Benthos will use a shared credentials file when connecting to AWS
    31  services. It's also possible to set them explicitly at the component level,
    32  allowing you to transfer data across accounts. You can find out more
    33  [in this document](/docs/guides/cloud/aws).`,
    34  		Async:   true,
    35  		Batches: true,
    36  		FieldSpecs: docs.FieldSpecs{
    37  			docs.FieldCommon("stream", "The stream to publish messages to."),
    38  			docs.FieldCommon("partition_key", "A required key for partitioning messages.").IsInterpolated(),
    39  			docs.FieldAdvanced("hash_key", "A optional hash key for partitioning messages.").IsInterpolated(),
    40  			docs.FieldCommon("max_in_flight", "The maximum number of messages to have in flight at a given time. Increase this to improve throughput."),
    41  			batch.FieldSpec(),
    42  		}.Merge(session.FieldSpecs()).Merge(retries.FieldSpecs()),
    43  		Categories: []Category{
    44  			CategoryServices,
    45  			CategoryAWS,
    46  		},
    47  	}
    48  
    49  	Constructors[TypeKinesis] = TypeSpec{
    50  		constructor: fromSimpleConstructor(NewKinesis),
    51  		Status:      docs.StatusDeprecated,
    52  		Summary: `
    53  Sends messages to a Kinesis stream.`,
    54  		Description: `
    55  ## Alternatives
    56  
    57  This output has been renamed to ` + "[`aws_kinesis`](/docs/components/outputs/aws_kinesis)" + `.
    58  
    59  Both the ` + "`partition_key`" + `(required) and ` + "`hash_key`" + ` (optional)
    60  fields can be dynamically set using function interpolations described
    61  [here](/docs/configuration/interpolation#bloblang-queries). When sending batched messages the
    62  interpolations are performed per message part.
    63  
    64  ### Credentials
    65  
    66  By default Benthos will use a shared credentials file when connecting to AWS
    67  services. It's also possible to set them explicitly at the component level,
    68  allowing you to transfer data across accounts. You can find out more
    69  [in this document](/docs/guides/cloud/aws).`,
    70  		Async:   true,
    71  		Batches: true,
    72  		FieldSpecs: docs.FieldSpecs{
    73  			docs.FieldCommon("stream", "The stream to publish messages to."),
    74  			docs.FieldCommon("partition_key", "A required key for partitioning messages.").IsInterpolated(),
    75  			docs.FieldAdvanced("hash_key", "A optional hash key for partitioning messages.").IsInterpolated(),
    76  			docs.FieldCommon("max_in_flight", "The maximum number of messages to have in flight at a given time. Increase this to improve throughput."),
    77  			batch.FieldSpec(),
    78  		}.Merge(session.FieldSpecs()).Merge(retries.FieldSpecs()),
    79  		Categories: []Category{
    80  			CategoryServices,
    81  			CategoryAWS,
    82  		},
    83  	}
    84  }
    85  
    86  //------------------------------------------------------------------------------
    87  
    88  // NewAWSKinesis creates a new Kinesis output type.
    89  func NewAWSKinesis(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    90  	return newKinesis(TypeAWSKinesis, conf.AWSKinesis, mgr, log, stats)
    91  }
    92  
    93  // NewKinesis creates a new Kinesis output type.
    94  func NewKinesis(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    95  	return newKinesis(TypeKinesis, conf.Kinesis, mgr, log, stats)
    96  }
    97  
    98  func newKinesis(name string, conf writer.KinesisConfig, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    99  	kin, err := writer.NewKinesisV2(conf, mgr, log, stats)
   100  	if err != nil {
   101  		return nil, err
   102  	}
   103  	var w Type
   104  	if conf.MaxInFlight == 1 {
   105  		w, err = NewWriter(name, kin, log, stats)
   106  	} else {
   107  		w, err = NewAsyncWriter(name, conf.MaxInFlight, kin, log, stats)
   108  	}
   109  	if err != nil {
   110  		return w, err
   111  	}
   112  	return NewBatcherFromConfig(conf.Batching, w, mgr, log, stats)
   113  }
   114  
   115  //------------------------------------------------------------------------------