github.com/Jeffail/benthos/v3@v3.65.0/lib/input/kinesis_balanced.go (about)

     1  package input
     2  
     3  import (
     4  	"github.com/Jeffail/benthos/v3/internal/docs"
     5  	"github.com/Jeffail/benthos/v3/lib/input/reader"
     6  	"github.com/Jeffail/benthos/v3/lib/log"
     7  	"github.com/Jeffail/benthos/v3/lib/message/batch"
     8  	"github.com/Jeffail/benthos/v3/lib/metrics"
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  	"github.com/Jeffail/benthos/v3/lib/util/aws/session"
    11  )
    12  
    13  //------------------------------------------------------------------------------
    14  
    15  func init() {
    16  	Constructors[TypeKinesisBalanced] = TypeSpec{
    17  		constructor: fromSimpleConstructor(NewKinesisBalanced),
    18  		Status:      docs.StatusDeprecated,
    19  		Summary: `
    20  Receives messages from a Kinesis stream and automatically balances shards across
    21  consumers.`,
    22  		Description: `
    23  ## Alternatives
    24  
    25  This input is being replaced with the shiny new ` + "[`aws_kinesis` input](/docs/components/inputs/aws_kinesis)" + `, which has improved features, consider trying it out instead.
    26  
    27  ### Metadata
    28  
    29  This input adds the following metadata fields to each message:
    30  
    31  ` + "```text" + `
    32  - kinesis_shard
    33  - kinesis_partition_key
    34  - kinesis_sequence_number
    35  ` + "```" + `
    36  
    37  You can access these metadata fields using
    38  [function interpolation](/docs/configuration/interpolation#metadata).`,
    39  		FieldSpecs: append(
    40  			append(docs.FieldSpecs{
    41  				docs.FieldCommon("stream", "The Kinesis stream to consume from."),
    42  				docs.FieldCommon("dynamodb_table", "A DynamoDB table to use for offset storage."),
    43  				docs.FieldAdvanced("dynamodb_billing_mode", "A billing mode to set for the offset DynamoDB table."),
    44  				docs.FieldAdvanced("dynamodb_read_provision", "The read capacity of the offset DynamoDB table."),
    45  				docs.FieldAdvanced("dynamodb_write_provision", "The write capacity of the offset DynamoDB table."),
    46  				docs.FieldCommon("start_from_oldest", "Whether to consume from the oldest message when an offset does not yet exist for the stream."),
    47  			}, session.FieldSpecs()...),
    48  			batch.FieldSpec(),
    49  			docs.FieldDeprecated("max_batch_count"),
    50  		),
    51  		Categories: []Category{
    52  			CategoryServices,
    53  			CategoryAWS,
    54  		},
    55  	}
    56  }
    57  
    58  //------------------------------------------------------------------------------
    59  
    60  // NewKinesisBalanced creates a new AWS KinesisBalanced input type.
    61  func NewKinesisBalanced(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    62  	// TODO: V4 Remove this.
    63  	if conf.KinesisBalanced.MaxBatchCount > 1 {
    64  		log.Warnf("Field '%v.max_batch_count' is deprecated, use '%v.batching.count' instead.\n", conf.Type, conf.Type)
    65  		conf.KinesisBalanced.Batching.Count = conf.KinesisBalanced.MaxBatchCount
    66  	}
    67  	var k reader.Async
    68  	var err error
    69  	if k, err = reader.NewKinesisBalanced(conf.KinesisBalanced, log, stats); err != nil {
    70  		return nil, err
    71  	}
    72  	if k, err = reader.NewAsyncBatcher(conf.KinesisBalanced.Batching, k, mgr, log, stats); err != nil {
    73  		return nil, err
    74  	}
    75  	k = reader.NewAsyncBundleUnacks(reader.NewAsyncPreserver(k))
    76  	return NewAsyncReader(TypeKinesisBalanced, true, k, log, stats)
    77  }
    78  
    79  //------------------------------------------------------------------------------