github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/redis.go (about) 1 package processor 2 3 import ( 4 "fmt" 5 "strconv" 6 "time" 7 8 "github.com/Jeffail/benthos/v3/internal/bloblang/field" 9 "github.com/Jeffail/benthos/v3/internal/docs" 10 bredis "github.com/Jeffail/benthos/v3/internal/impl/redis" 11 "github.com/Jeffail/benthos/v3/internal/interop" 12 "github.com/Jeffail/benthos/v3/internal/tracing" 13 "github.com/Jeffail/benthos/v3/lib/log" 14 "github.com/Jeffail/benthos/v3/lib/metrics" 15 "github.com/Jeffail/benthos/v3/lib/types" 16 "github.com/go-redis/redis/v7" 17 ) 18 19 //------------------------------------------------------------------------------ 20 21 func init() { 22 Constructors[TypeRedis] = TypeSpec{ 23 constructor: NewRedis, 24 Categories: []Category{ 25 CategoryIntegration, 26 }, 27 Summary: ` 28 Performs actions against Redis that aren't possible using a 29 ` + "[`cache`](/docs/components/processors/cache)" + ` processor. Actions are 30 performed for each message of a batch, where the contents are replaced with the 31 result.`, 32 Description: ` 33 ## Operators 34 35 ### ` + "`keys`" + ` 36 37 Returns an array of strings containing all the keys that match the pattern specified by the ` + "`key` field" + `. 38 39 ### ` + "`scard`" + ` 40 41 Returns the cardinality of a set, or ` + "`0`" + ` if the key does not exist. 42 43 ### ` + "`sadd`" + ` 44 45 Adds a new member to a set. Returns ` + "`1`" + ` if the member was added. 46 47 ### ` + "`incrby`" + ` 48 49 Increments the number stored at ` + "`key`" + ` by the message content. If the 50 key does not exist, it is set to ` + "`0`" + ` before performing the operation. 51 Returns the value of ` + "`key`" + ` after the increment.`, 52 FieldSpecs: bredis.ConfigDocs().Add( 53 docs.FieldCommon("operator", "The [operator](#operators) to apply.").HasOptions("scard", "sadd", "incrby", "keys"), 54 docs.FieldCommon("key", "A key to use for the target operator.").IsInterpolated(), 55 docs.FieldAdvanced("retries", "The maximum number of retries before abandoning a request."), 56 docs.FieldAdvanced("retry_period", "The time to wait before consecutive retry attempts."), 57 PartsFieldSpec, 58 ), 59 Examples: []docs.AnnotatedExample{ 60 { 61 Title: "Querying Cardinality", 62 Summary: ` 63 If given payloads containing a metadata field ` + "`set_key`" + ` it's possible 64 to query and store the cardinality of the set for each message using a 65 ` + "[`branch` processor](/docs/components/processors/branch)" + ` in order to 66 augment rather than replace the message contents:`, 67 Config: ` 68 pipeline: 69 processors: 70 - branch: 71 processors: 72 - redis: 73 url: TODO 74 operator: scard 75 key: ${! meta("set_key") } 76 result_map: 'root.cardinality = this' 77 `, 78 }, 79 { 80 Title: "Running Total", 81 Summary: ` 82 If we have JSON data containing number of friends visited during covid 19: 83 84 ` + "```json" + ` 85 {"name":"ash","month":"feb","year":2019,"friends_visited":10} 86 {"name":"ash","month":"apr","year":2019,"friends_visited":-2} 87 {"name":"bob","month":"feb","year":2019,"friends_visited":3} 88 {"name":"bob","month":"apr","year":2019,"friends_visited":1} 89 ` + "```" + ` 90 91 We can add a field that contains the running total number of friends visited: 92 93 ` + "```json" + ` 94 {"name":"ash","month":"feb","year":2019,"friends_visited":10,"total":10} 95 {"name":"ash","month":"apr","year":2019,"friends_visited":-2,"total":8} 96 {"name":"bob","month":"feb","year":2019,"friends_visited":3,"total":3} 97 {"name":"bob","month":"apr","year":2019,"friends_visited":1,"total":4} 98 ` + "```" + ` 99 100 Using the ` + "`incrby`" + ` operator: 101 `, 102 Config: ` 103 pipeline: 104 processors: 105 - branch: 106 request_map: | 107 root = this.friends_visited 108 meta name = this.name 109 processors: 110 - redis: 111 url: TODO 112 operator: incrby 113 key: ${! meta("name") } 114 result_map: 'root.total = this' 115 `, 116 }, 117 }, 118 } 119 } 120 121 //------------------------------------------------------------------------------ 122 123 // RedisConfig contains configuration fields for the Redis processor. 124 type RedisConfig struct { 125 bredis.Config `json:",inline" yaml:",inline"` 126 Parts []int `json:"parts" yaml:"parts"` 127 Operator string `json:"operator" yaml:"operator"` 128 Key string `json:"key" yaml:"key"` 129 Retries int `json:"retries" yaml:"retries"` 130 RetryPeriod string `json:"retry_period" yaml:"retry_period"` 131 } 132 133 // NewRedisConfig returns a RedisConfig with default values. 134 func NewRedisConfig() RedisConfig { 135 return RedisConfig{ 136 Config: bredis.NewConfig(), 137 Parts: []int{}, 138 Operator: "scard", 139 Key: "", 140 Retries: 3, 141 RetryPeriod: "500ms", 142 } 143 } 144 145 //------------------------------------------------------------------------------ 146 147 // Redis is a processor that performs redis operations 148 type Redis struct { 149 parts []int 150 conf Config 151 log log.Modular 152 stats metrics.Type 153 154 key *field.Expression 155 156 operator redisOperator 157 client redis.UniversalClient 158 retryPeriod time.Duration 159 160 mCount metrics.StatCounter 161 mErr metrics.StatCounter 162 mSent metrics.StatCounter 163 mBatchSent metrics.StatCounter 164 mRedisRetry metrics.StatCounter 165 } 166 167 // NewRedis returns a Redis processor. 168 func NewRedis( 169 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 170 ) (Type, error) { 171 var retryPeriod time.Duration 172 if tout := conf.Redis.RetryPeriod; len(tout) > 0 { 173 var err error 174 if retryPeriod, err = time.ParseDuration(tout); err != nil { 175 return nil, fmt.Errorf("failed to parse retry period string: %v", err) 176 } 177 } 178 179 client, err := conf.Redis.Config.Client() 180 if err != nil { 181 return nil, err 182 } 183 184 key, err := interop.NewBloblangField(mgr, conf.Redis.Key) 185 if err != nil { 186 return nil, fmt.Errorf("failed to parse key expression: %v", err) 187 } 188 189 r := &Redis{ 190 parts: conf.Redis.Parts, 191 conf: conf, 192 log: log, 193 stats: stats, 194 195 key: key, 196 197 retryPeriod: retryPeriod, 198 client: client, 199 200 mCount: stats.GetCounter("count"), 201 mErr: stats.GetCounter("error"), 202 mSent: stats.GetCounter("sent"), 203 mBatchSent: stats.GetCounter("batch.sent"), 204 mRedisRetry: stats.GetCounter("redis.retry"), 205 } 206 207 if r.operator, err = getRedisOperator(conf.Redis.Operator); err != nil { 208 return nil, err 209 } 210 return r, nil 211 } 212 213 //------------------------------------------------------------------------------ 214 215 type redisOperator func(r *Redis, key string, part types.Part) error 216 217 func newRedisKeysOperator() redisOperator { 218 return func(r *Redis, key string, part types.Part) error { 219 res, err := r.client.Keys(key).Result() 220 221 for i := 0; i <= r.conf.Redis.Retries && err != nil; i++ { 222 r.log.Errorf("Keys command failed: %v\n", err) 223 <-time.After(r.retryPeriod) 224 r.mRedisRetry.Incr(1) 225 res, err = r.client.Keys(key).Result() 226 } 227 if err != nil { 228 return err 229 } 230 231 iRes := make([]interface{}, 0, len(res)) 232 for _, v := range res { 233 iRes = append(iRes, v) 234 } 235 return part.SetJSON(iRes) 236 } 237 } 238 239 func newRedisSCardOperator() redisOperator { 240 return func(r *Redis, key string, part types.Part) error { 241 res, err := r.client.SCard(key).Result() 242 243 for i := 0; i <= r.conf.Redis.Retries && err != nil; i++ { 244 r.log.Errorf("SCard command failed: %v\n", err) 245 <-time.After(r.retryPeriod) 246 r.mRedisRetry.Incr(1) 247 res, err = r.client.SCard(key).Result() 248 } 249 if err != nil { 250 return err 251 } 252 253 part.Set(strconv.AppendInt(nil, res, 10)) 254 return nil 255 } 256 } 257 258 func newRedisSAddOperator() redisOperator { 259 return func(r *Redis, key string, part types.Part) error { 260 res, err := r.client.SAdd(key, part.Get()).Result() 261 262 for i := 0; i <= r.conf.Redis.Retries && err != nil; i++ { 263 r.log.Errorf("SAdd command failed: %v\n", err) 264 <-time.After(r.retryPeriod) 265 r.mRedisRetry.Incr(1) 266 res, err = r.client.SAdd(key, part.Get()).Result() 267 } 268 if err != nil { 269 return err 270 } 271 272 part.Set(strconv.AppendInt(nil, res, 10)) 273 return nil 274 } 275 } 276 277 func newRedisIncrByOperator() redisOperator { 278 return func(r *Redis, key string, part types.Part) error { 279 valueInt, err := strconv.Atoi(string(part.Get())) 280 if err != nil { 281 return err 282 } 283 res, err := r.client.IncrBy(key, int64(valueInt)).Result() 284 285 for i := 0; i <= r.conf.Redis.Retries && err != nil; i++ { 286 r.log.Errorf("incrby command failed: %v\n", err) 287 <-time.After(r.retryPeriod) 288 r.mRedisRetry.Incr(1) 289 res, err = r.client.IncrBy(key, int64(valueInt)).Result() 290 } 291 if err != nil { 292 return err 293 } 294 295 part.Set(strconv.AppendInt(nil, res, 10)) 296 return nil 297 } 298 } 299 300 func getRedisOperator(opStr string) (redisOperator, error) { 301 switch opStr { 302 case "keys": 303 return newRedisKeysOperator(), nil 304 case "sadd": 305 return newRedisSAddOperator(), nil 306 case "scard": 307 return newRedisSCardOperator(), nil 308 case "incrby": 309 return newRedisIncrByOperator(), nil 310 } 311 return nil, fmt.Errorf("operator not recognised: %v", opStr) 312 } 313 314 // ProcessMessage applies the processor to a message, either creating >0 315 // resulting messages or a response to be sent back to the message source. 316 func (r *Redis) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 317 r.mCount.Incr(1) 318 newMsg := msg.Copy() 319 320 proc := func(index int, span *tracing.Span, part types.Part) error { 321 key := r.key.String(index, newMsg) 322 if err := r.operator(r, key, part); err != nil { 323 r.mErr.Incr(1) 324 r.log.Debugf("Operator failed for key '%s': %v\n", key, err) 325 return err 326 } 327 return nil 328 } 329 330 IteratePartsWithSpanV2(TypeRedis, r.parts, newMsg, proc) 331 332 r.mBatchSent.Incr(1) 333 r.mSent.Incr(int64(newMsg.Len())) 334 return []types.Message{newMsg}, nil 335 } 336 337 // CloseAsync shuts down the processor and stops processing requests. 338 func (r *Redis) CloseAsync() { 339 } 340 341 // WaitForClose blocks until the processor has closed down. 342 func (r *Redis) WaitForClose(timeout time.Duration) error { 343 r.client.Close() 344 return nil 345 } 346 347 //------------------------------------------------------------------------------