github.com/Jeffail/benthos/v3@v3.65.0/lib/input/azure_blob_storage_config.go (about)

     1  package input
     2  
     3  import (
     4  	"github.com/Jeffail/benthos/v3/internal/codec"
     5  	"github.com/Jeffail/benthos/v3/internal/docs"
     6  	"github.com/Jeffail/benthos/v3/lib/input/reader"
     7  	"github.com/Jeffail/benthos/v3/lib/log"
     8  	"github.com/Jeffail/benthos/v3/lib/metrics"
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  )
    11  
    12  func init() {
    13  	Constructors[TypeAzureBlobStorage] = TypeSpec{
    14  		constructor: fromSimpleConstructor(func(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    15  			r, err := newAzureBlobStorage(conf.AzureBlobStorage, log, stats)
    16  			if err != nil {
    17  				return nil, err
    18  			}
    19  			return NewAsyncReader(
    20  				TypeAzureBlobStorage,
    21  				true,
    22  				reader.NewAsyncBundleUnacks(
    23  					reader.NewAsyncPreserver(r),
    24  				),
    25  				log, stats,
    26  			)
    27  		}),
    28  		Status:  docs.StatusBeta,
    29  		Version: "3.36.0",
    30  		Summary: `
    31  Downloads objects within an Azure Blob Storage container, optionally filtered by
    32  a prefix.`,
    33  		Description: `
    34  Downloads objects within an Azure Blob Storage container, optionally filtered by a prefix.
    35  
    36  ## Downloading Large Files
    37  
    38  When downloading large files it's often necessary to process it in streamed parts in order to avoid loading the entire file in memory at a given time. In order to do this a ` + "[`codec`](#codec)" + ` can be specified that determines how to break the input into smaller individual messages.
    39  
    40  ## Metadata
    41  
    42  This input adds the following metadata fields to each message:
    43  
    44  ` + "```" + `
    45  - blob_storage_key
    46  - blob_storage_container
    47  - blob_storage_last_modified
    48  - blob_storage_last_modified_unix
    49  - blob_storage_content_type
    50  - blob_storage_content_encoding
    51  - All user defined metadata
    52  ` + "```" + `
    53  
    54  You can access these metadata fields using [function interpolation](/docs/configuration/interpolation#metadata).`,
    55  		FieldSpecs: docs.FieldSpecs{
    56  			docs.FieldCommon(
    57  				"storage_account",
    58  				"The storage account to download blobs from. This field is ignored if `storage_connection_string` is set.",
    59  			),
    60  			docs.FieldCommon(
    61  				"storage_access_key",
    62  				"The storage account access key. This field is ignored if `storage_connection_string` is set.",
    63  			),
    64  			docs.FieldCommon(
    65  				"storage_sas_token",
    66  				"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.",
    67  			).AtVersion("3.38.0"),
    68  			docs.FieldCommon(
    69  				"storage_connection_string",
    70  				"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.",
    71  			),
    72  			docs.FieldCommon(
    73  				"container", "The name of the container from which to download blobs.",
    74  			),
    75  			docs.FieldCommon("prefix", "An optional path prefix, if set only objects with the prefix are consumed."),
    76  			codec.ReaderDocs,
    77  			docs.FieldAdvanced("delete_objects", "Whether to delete downloaded objects from the blob once they are processed."),
    78  		},
    79  		Categories: []Category{
    80  			CategoryServices,
    81  			CategoryAzure,
    82  		},
    83  	}
    84  }
    85  
    86  //------------------------------------------------------------------------------
    87  
    88  // AzureBlobStorageConfig contains configuration fields for the AzureBlobStorage
    89  // input type.
    90  type AzureBlobStorageConfig struct {
    91  	StorageAccount          string `json:"storage_account" yaml:"storage_account"`
    92  	StorageAccessKey        string `json:"storage_access_key" yaml:"storage_access_key"`
    93  	StorageSASToken         string `json:"storage_sas_token" yaml:"storage_sas_token"`
    94  	StorageConnectionString string `json:"storage_connection_string" yaml:"storage_connection_string"`
    95  	Container               string `json:"container" yaml:"container"`
    96  	Prefix                  string `json:"prefix" yaml:"prefix"`
    97  	Codec                   string `json:"codec" yaml:"codec"`
    98  	DeleteObjects           bool   `json:"delete_objects" yaml:"delete_objects"`
    99  }
   100  
   101  // NewAzureBlobStorageConfig creates a new AzureBlobStorageConfig with default
   102  // values.
   103  func NewAzureBlobStorageConfig() AzureBlobStorageConfig {
   104  	return AzureBlobStorageConfig{
   105  		Codec: "all-bytes",
   106  	}
   107  }