github.com/Jeffail/benthos/v3@v3.65.0/lib/output/writer/redis_list.go (about) 1 package writer 2 3 import ( 4 "context" 5 "fmt" 6 "sync" 7 "time" 8 9 ibatch "github.com/Jeffail/benthos/v3/internal/batch" 10 "github.com/Jeffail/benthos/v3/internal/bloblang/field" 11 bredis "github.com/Jeffail/benthos/v3/internal/impl/redis" 12 "github.com/Jeffail/benthos/v3/internal/interop" 13 "github.com/Jeffail/benthos/v3/lib/log" 14 "github.com/Jeffail/benthos/v3/lib/message/batch" 15 "github.com/Jeffail/benthos/v3/lib/metrics" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 "github.com/go-redis/redis/v7" 18 ) 19 20 //------------------------------------------------------------------------------ 21 22 // RedisListConfig contains configuration fields for the RedisList output type. 23 type RedisListConfig struct { 24 bredis.Config `json:",inline" yaml:",inline"` 25 Key string `json:"key" yaml:"key"` 26 MaxInFlight int `json:"max_in_flight" yaml:"max_in_flight"` 27 Batching batch.PolicyConfig `json:"batching" yaml:"batching"` 28 } 29 30 // NewRedisListConfig creates a new RedisListConfig with default values. 31 func NewRedisListConfig() RedisListConfig { 32 return RedisListConfig{ 33 Config: bredis.NewConfig(), 34 Key: "benthos_list", 35 MaxInFlight: 1, 36 Batching: batch.NewPolicyConfig(), 37 } 38 } 39 40 //------------------------------------------------------------------------------ 41 42 // RedisList is an output type that serves RedisList messages. 43 type RedisList struct { 44 log log.Modular 45 stats metrics.Type 46 47 conf RedisListConfig 48 49 keyStr *field.Expression 50 51 client redis.UniversalClient 52 connMut sync.RWMutex 53 } 54 55 // NewRedisList creates a new RedisList output type. 56 // 57 // Deprecated: use the V2 API instead. 58 func NewRedisList( 59 conf RedisListConfig, 60 log log.Modular, 61 stats metrics.Type, 62 ) (*RedisList, error) { 63 return NewRedisListV2(conf, types.NoopMgr(), log, stats) 64 } 65 66 // NewRedisListV2 creates a new RedisList output type. 67 func NewRedisListV2( 68 conf RedisListConfig, 69 mgr types.Manager, 70 log log.Modular, 71 stats metrics.Type, 72 ) (*RedisList, error) { 73 r := &RedisList{ 74 log: log, 75 stats: stats, 76 conf: conf, 77 } 78 79 var err error 80 if r.keyStr, err = interop.NewBloblangField(mgr, conf.Key); err != nil { 81 return nil, fmt.Errorf("failed to parse key expression: %v", err) 82 } 83 if _, err := conf.Config.Client(); err != nil { 84 return nil, err 85 } 86 87 return r, nil 88 } 89 90 //------------------------------------------------------------------------------ 91 92 // ConnectWithContext establishes a connection to an RedisList server. 93 func (r *RedisList) ConnectWithContext(ctx context.Context) error { 94 return r.Connect() 95 } 96 97 // Connect establishes a connection to an RedisList server. 98 func (r *RedisList) Connect() error { 99 r.connMut.Lock() 100 defer r.connMut.Unlock() 101 102 client, err := r.conf.Config.Client() 103 if err != nil { 104 return err 105 } 106 if _, err = client.Ping().Result(); err != nil { 107 return err 108 } 109 110 r.client = client 111 return nil 112 } 113 114 //------------------------------------------------------------------------------ 115 116 // WriteWithContext attempts to write a message by pushing it to the end of a 117 // Redis list. 118 func (r *RedisList) WriteWithContext(ctx context.Context, msg types.Message) error { 119 r.connMut.RLock() 120 client := r.client 121 r.connMut.RUnlock() 122 123 if client == nil { 124 return types.ErrNotConnected 125 } 126 127 if msg.Len() == 1 { 128 key := r.keyStr.String(0, msg) 129 if err := client.RPush(key, msg.Get(0).Get()).Err(); err != nil { 130 r.disconnect() 131 r.log.Errorf("Error from redis: %v\n", err) 132 return types.ErrNotConnected 133 } 134 return nil 135 } 136 137 pipe := client.Pipeline() 138 msg.Iter(func(i int, p types.Part) error { 139 key := r.keyStr.String(0, msg) 140 _ = pipe.RPush(key, p.Get()) 141 return nil 142 }) 143 cmders, err := pipe.Exec() 144 if err != nil { 145 r.disconnect() 146 r.log.Errorf("Error from redis: %v\n", err) 147 return types.ErrNotConnected 148 } 149 150 var batchErr *ibatch.Error 151 for i, res := range cmders { 152 if res.Err() != nil { 153 if batchErr == nil { 154 batchErr = ibatch.NewError(msg, res.Err()) 155 } 156 batchErr.Failed(i, res.Err()) 157 } 158 } 159 if batchErr != nil { 160 return batchErr 161 } 162 return nil 163 } 164 165 // Write attempts to write a message by pushing it to the end of a Redis list. 166 func (r *RedisList) Write(msg types.Message) error { 167 return r.WriteWithContext(context.Background(), msg) 168 } 169 170 // disconnect safely closes a connection to an RedisList server. 171 func (r *RedisList) disconnect() error { 172 r.connMut.Lock() 173 defer r.connMut.Unlock() 174 if r.client != nil { 175 err := r.client.Close() 176 r.client = nil 177 return err 178 } 179 return nil 180 } 181 182 // CloseAsync shuts down the RedisList output and stops processing messages. 183 func (r *RedisList) CloseAsync() { 184 go r.disconnect() 185 } 186 187 // WaitForClose blocks until the RedisList output has closed down. 188 func (r *RedisList) WaitForClose(timeout time.Duration) error { 189 return nil 190 } 191 192 //------------------------------------------------------------------------------