github.com/Jeffail/benthos/v3@v3.65.0/lib/output/writer/azure_table_storage.go (about) 1 //go:build !wasm 2 // +build !wasm 3 4 package writer 5 6 import ( 7 "context" 8 "encoding/json" 9 "errors" 10 "fmt" 11 "strings" 12 "time" 13 14 "github.com/Azure/azure-sdk-for-go/storage" 15 "github.com/Jeffail/benthos/v3/internal/bloblang/field" 16 "github.com/Jeffail/benthos/v3/internal/interop" 17 "github.com/Jeffail/benthos/v3/lib/log" 18 "github.com/Jeffail/benthos/v3/lib/metrics" 19 "github.com/Jeffail/benthos/v3/lib/types" 20 ) 21 22 //------------------------------------------------------------------------------ 23 24 // AzureTableStorage is a benthos writer. Type implementation that writes messages to an 25 // Azure Table Storage table. 26 type AzureTableStorage struct { 27 conf AzureTableStorageConfig 28 tableName *field.Expression 29 partitionKey *field.Expression 30 rowKey *field.Expression 31 properties map[string]*field.Expression 32 client storage.TableServiceClient 33 timeout time.Duration 34 log log.Modular 35 stats metrics.Type 36 } 37 38 // NewAzureTableStorage creates a new Azure Table Storage writer Type. 39 // 40 // Deprecated: use the V2 API instead. 41 func NewAzureTableStorage( 42 conf AzureTableStorageConfig, 43 log log.Modular, 44 stats metrics.Type, 45 ) (*AzureTableStorage, error) { 46 return NewAzureTableStorageV2(conf, types.NoopMgr(), log, stats) 47 } 48 49 // NewAzureTableStorageV2 creates a new Azure Table Storage writer Type. 50 func NewAzureTableStorageV2( 51 conf AzureTableStorageConfig, 52 mgr types.Manager, 53 log log.Modular, 54 stats metrics.Type, 55 ) (*AzureTableStorage, error) { 56 var timeout time.Duration 57 var err error 58 if tout := conf.Timeout; len(tout) > 0 { 59 if timeout, err = time.ParseDuration(tout); err != nil { 60 return nil, fmt.Errorf("failed to parse timeout period string: %v", err) 61 } 62 } 63 if conf.StorageAccount == "" && conf.StorageConnectionString == "" { 64 return nil, errors.New("invalid azure storage account credentials") 65 } 66 var client storage.Client 67 if conf.StorageConnectionString != "" { 68 if strings.Contains(conf.StorageConnectionString, "UseDevelopmentStorage=true;") { 69 client, err = storage.NewEmulatorClient() 70 } else { 71 client, err = storage.NewClientFromConnectionString(conf.StorageConnectionString) 72 } 73 } else { 74 client, err = storage.NewBasicClient(conf.StorageAccount, conf.StorageAccessKey) 75 } 76 if err != nil { 77 return nil, fmt.Errorf("invalid azure storage account credentials: %v", err) 78 } 79 a := &AzureTableStorage{ 80 conf: conf, 81 log: log, 82 stats: stats, 83 timeout: timeout, 84 client: client.GetTableService(), 85 } 86 if a.tableName, err = interop.NewBloblangField(mgr, conf.TableName); err != nil { 87 return nil, fmt.Errorf("failed to parse table name expression: %v", err) 88 } 89 if a.partitionKey, err = interop.NewBloblangField(mgr, conf.PartitionKey); err != nil { 90 return nil, fmt.Errorf("failed to parse partition key expression: %v", err) 91 } 92 if a.rowKey, err = interop.NewBloblangField(mgr, conf.RowKey); err != nil { 93 return nil, fmt.Errorf("failed to parse row key expression: %v", err) 94 } 95 a.properties = make(map[string]*field.Expression) 96 for property, value := range conf.Properties { 97 if a.properties[property], err = interop.NewBloblangField(mgr, value); err != nil { 98 return nil, fmt.Errorf("failed to parse property expression: %v", err) 99 } 100 } 101 102 return a, nil 103 } 104 105 // ConnectWithContext attempts to establish a connection to the target Table Storage Account. 106 func (a *AzureTableStorage) ConnectWithContext(ctx context.Context) error { 107 return a.Connect() 108 } 109 110 // Connect attempts to establish a connection to the target Table Storage Account. 111 func (a *AzureTableStorage) Connect() error { 112 return nil 113 } 114 115 // Write attempts to write message contents to a target Azure Table Storage container as files. 116 func (a *AzureTableStorage) Write(msg types.Message) error { 117 return a.WriteWithContext(context.Background(), msg) 118 } 119 120 // WriteWithContext attempts to write message contents to a target storage account as files. 121 func (a *AzureTableStorage) WriteWithContext(wctx context.Context, msg types.Message) error { 122 writeReqs := make(map[string]map[string][]*storage.Entity) 123 if err := IterateBatchedSend(msg, func(i int, p types.Part) error { 124 entity := &storage.Entity{} 125 tableName := a.tableName.String(i, msg) 126 partitionKey := a.partitionKey.String(i, msg) 127 entity.PartitionKey = a.partitionKey.String(i, msg) 128 entity.RowKey = a.rowKey.String(i, msg) 129 entity.Properties = a.getProperties(i, p, msg) 130 if writeReqs[tableName] == nil { 131 writeReqs[tableName] = make(map[string][]*storage.Entity) 132 } 133 writeReqs[tableName][partitionKey] = append(writeReqs[tableName][partitionKey], entity) 134 return nil 135 }); err != nil { 136 return err 137 } 138 return a.writeBatches(writeReqs) 139 } 140 141 func (a *AzureTableStorage) getProperties(i int, p types.Part, msg types.Message) map[string]interface{} { 142 properties := make(map[string]interface{}) 143 if len(a.properties) == 0 { 144 err := json.Unmarshal(p.Get(), &properties) 145 if err != nil { 146 a.log.Errorf("error unmarshalling message: %v.", err) 147 } 148 for property, v := range properties { 149 switch v.(type) { 150 case []interface{}, map[string]interface{}: 151 m, err := json.Marshal(v) 152 if err != nil { 153 a.log.Errorf("error marshaling property: %v.", property) 154 } 155 properties[property] = string(m) 156 } 157 } 158 } else { 159 for property, value := range a.properties { 160 properties[property] = value.String(i, msg) 161 } 162 } 163 return properties 164 } 165 166 func (a *AzureTableStorage) writeBatches(writeReqs map[string]map[string][]*storage.Entity) error { 167 for tn, pks := range writeReqs { 168 table := a.client.GetTableReference(tn) 169 for _, entities := range pks { 170 tableBatch := table.NewBatch() 171 ne := len(entities) 172 for i, entity := range entities { 173 entity.Table = table 174 if err := a.addToBatch(tableBatch, a.conf.InsertType, entity); err != nil { 175 return err 176 } 177 if reachedBatchLimit(i) || isLastEntity(i, ne) { 178 if err := a.executeBatch(table, tableBatch); err != nil { 179 return err 180 } 181 tableBatch = table.NewBatch() 182 } 183 } 184 } 185 } 186 return nil 187 } 188 189 func (a *AzureTableStorage) executeBatch(table *storage.Table, tableBatch *storage.TableBatch) error { 190 if err := tableBatch.ExecuteBatch(); err != nil { 191 if tableDoesNotExist(err) { 192 if cerr := table.Create(uint(10), storage.FullMetadata, nil); cerr != nil { 193 return cerr 194 } 195 err = tableBatch.ExecuteBatch() 196 } 197 return err 198 } 199 return nil 200 } 201 202 func tableDoesNotExist(err error) bool { 203 if cerr, ok := err.(storage.AzureStorageServiceError); ok { 204 return cerr.Code == "TableNotFound" 205 } 206 return false 207 } 208 209 func isLastEntity(i, ne int) bool { 210 return i+1 == ne 211 } 212 213 func reachedBatchLimit(i int) bool { 214 const batchSizeLimit = 100 215 return (i+1)%batchSizeLimit == 0 216 } 217 218 func (a *AzureTableStorage) addToBatch(tableBatch *storage.TableBatch, insertType string, entity *storage.Entity) error { 219 switch insertType { 220 case "INSERT": 221 tableBatch.InsertEntity(entity) 222 case "INSERT_MERGE": 223 tableBatch.InsertOrMergeEntity(entity, true) 224 case "INSERT_REPLACE": 225 tableBatch.InsertOrReplaceEntity(entity, true) 226 default: 227 return fmt.Errorf("invalid insert type") 228 } 229 return nil 230 } 231 232 // CloseAsync begins cleaning up resources used by this reader asynchronously. 233 func (a *AzureTableStorage) CloseAsync() { 234 } 235 236 // WaitForClose will block until either the reader is closed or a specified 237 // timeout occurs. 238 func (a *AzureTableStorage) WaitForClose(time.Duration) error { 239 return nil 240 } 241 242 //------------------------------------------------------------------------------