github.com/Jeffail/benthos/v3@v3.65.0/lib/input/azure_blob_storage_config.go (about) 1 package input 2 3 import ( 4 "github.com/Jeffail/benthos/v3/internal/codec" 5 "github.com/Jeffail/benthos/v3/internal/docs" 6 "github.com/Jeffail/benthos/v3/lib/input/reader" 7 "github.com/Jeffail/benthos/v3/lib/log" 8 "github.com/Jeffail/benthos/v3/lib/metrics" 9 "github.com/Jeffail/benthos/v3/lib/types" 10 ) 11 12 func init() { 13 Constructors[TypeAzureBlobStorage] = TypeSpec{ 14 constructor: fromSimpleConstructor(func(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) { 15 r, err := newAzureBlobStorage(conf.AzureBlobStorage, log, stats) 16 if err != nil { 17 return nil, err 18 } 19 return NewAsyncReader( 20 TypeAzureBlobStorage, 21 true, 22 reader.NewAsyncBundleUnacks( 23 reader.NewAsyncPreserver(r), 24 ), 25 log, stats, 26 ) 27 }), 28 Status: docs.StatusBeta, 29 Version: "3.36.0", 30 Summary: ` 31 Downloads objects within an Azure Blob Storage container, optionally filtered by 32 a prefix.`, 33 Description: ` 34 Downloads objects within an Azure Blob Storage container, optionally filtered by a prefix. 35 36 ## Downloading Large Files 37 38 When downloading large files it's often necessary to process it in streamed parts in order to avoid loading the entire file in memory at a given time. In order to do this a ` + "[`codec`](#codec)" + ` can be specified that determines how to break the input into smaller individual messages. 39 40 ## Metadata 41 42 This input adds the following metadata fields to each message: 43 44 ` + "```" + ` 45 - blob_storage_key 46 - blob_storage_container 47 - blob_storage_last_modified 48 - blob_storage_last_modified_unix 49 - blob_storage_content_type 50 - blob_storage_content_encoding 51 - All user defined metadata 52 ` + "```" + ` 53 54 You can access these metadata fields using [function interpolation](/docs/configuration/interpolation#metadata).`, 55 FieldSpecs: docs.FieldSpecs{ 56 docs.FieldCommon( 57 "storage_account", 58 "The storage account to download blobs from. This field is ignored if `storage_connection_string` is set.", 59 ), 60 docs.FieldCommon( 61 "storage_access_key", 62 "The storage account access key. This field is ignored if `storage_connection_string` is set.", 63 ), 64 docs.FieldCommon( 65 "storage_sas_token", 66 "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.", 67 ).AtVersion("3.38.0"), 68 docs.FieldCommon( 69 "storage_connection_string", 70 "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.", 71 ), 72 docs.FieldCommon( 73 "container", "The name of the container from which to download blobs.", 74 ), 75 docs.FieldCommon("prefix", "An optional path prefix, if set only objects with the prefix are consumed."), 76 codec.ReaderDocs, 77 docs.FieldAdvanced("delete_objects", "Whether to delete downloaded objects from the blob once they are processed."), 78 }, 79 Categories: []Category{ 80 CategoryServices, 81 CategoryAzure, 82 }, 83 } 84 } 85 86 //------------------------------------------------------------------------------ 87 88 // AzureBlobStorageConfig contains configuration fields for the AzureBlobStorage 89 // input type. 90 type AzureBlobStorageConfig struct { 91 StorageAccount string `json:"storage_account" yaml:"storage_account"` 92 StorageAccessKey string `json:"storage_access_key" yaml:"storage_access_key"` 93 StorageSASToken string `json:"storage_sas_token" yaml:"storage_sas_token"` 94 StorageConnectionString string `json:"storage_connection_string" yaml:"storage_connection_string"` 95 Container string `json:"container" yaml:"container"` 96 Prefix string `json:"prefix" yaml:"prefix"` 97 Codec string `json:"codec" yaml:"codec"` 98 DeleteObjects bool `json:"delete_objects" yaml:"delete_objects"` 99 } 100 101 // NewAzureBlobStorageConfig creates a new AzureBlobStorageConfig with default 102 // values. 103 func NewAzureBlobStorageConfig() AzureBlobStorageConfig { 104 return AzureBlobStorageConfig{ 105 Codec: "all-bytes", 106 } 107 }