github.com/Jeffail/benthos/v3@v3.65.0/lib/output/writer/redis_pubsub.go (about) 1 package writer 2 3 import ( 4 "context" 5 "fmt" 6 "sync" 7 "time" 8 9 ibatch "github.com/Jeffail/benthos/v3/internal/batch" 10 "github.com/Jeffail/benthos/v3/internal/bloblang/field" 11 bredis "github.com/Jeffail/benthos/v3/internal/impl/redis" 12 "github.com/Jeffail/benthos/v3/internal/interop" 13 "github.com/Jeffail/benthos/v3/lib/log" 14 "github.com/Jeffail/benthos/v3/lib/message/batch" 15 "github.com/Jeffail/benthos/v3/lib/metrics" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 "github.com/go-redis/redis/v7" 18 ) 19 20 //------------------------------------------------------------------------------ 21 22 // RedisPubSubConfig contains configuration fields for the RedisPubSub output 23 // type. 24 type RedisPubSubConfig struct { 25 bredis.Config `json:",inline" yaml:",inline"` 26 Channel string `json:"channel" yaml:"channel"` 27 MaxInFlight int `json:"max_in_flight" yaml:"max_in_flight"` 28 Batching batch.PolicyConfig `json:"batching" yaml:"batching"` 29 } 30 31 // NewRedisPubSubConfig creates a new RedisPubSubConfig with default values. 32 func NewRedisPubSubConfig() RedisPubSubConfig { 33 return RedisPubSubConfig{ 34 Config: bredis.NewConfig(), 35 Channel: "benthos_chan", 36 MaxInFlight: 1, 37 Batching: batch.NewPolicyConfig(), 38 } 39 } 40 41 //------------------------------------------------------------------------------ 42 43 // RedisPubSub is an output type that serves RedisPubSub messages. 44 type RedisPubSub struct { 45 log log.Modular 46 stats metrics.Type 47 48 conf RedisPubSubConfig 49 channelStr *field.Expression 50 51 client redis.UniversalClient 52 connMut sync.RWMutex 53 } 54 55 // NewRedisPubSub creates a new RedisPubSub output type. 56 // 57 // Deprecated: use the V2 API instead. 58 func NewRedisPubSub( 59 conf RedisPubSubConfig, 60 log log.Modular, 61 stats metrics.Type, 62 ) (*RedisPubSub, error) { 63 return NewRedisPubSubV2(conf, types.NoopMgr(), log, stats) 64 } 65 66 // NewRedisPubSubV2 creates a new RedisPubSub output type. 67 func NewRedisPubSubV2( 68 conf RedisPubSubConfig, 69 mgr types.Manager, 70 log log.Modular, 71 stats metrics.Type, 72 ) (*RedisPubSub, error) { 73 r := &RedisPubSub{ 74 log: log, 75 stats: stats, 76 conf: conf, 77 } 78 var err error 79 if r.channelStr, err = interop.NewBloblangField(mgr, conf.Channel); err != nil { 80 return nil, fmt.Errorf("failed to parse channel expression: %v", err) 81 } 82 if _, err = conf.Config.Client(); err != nil { 83 return nil, err 84 } 85 return r, nil 86 } 87 88 //------------------------------------------------------------------------------ 89 90 // ConnectWithContext establishes a connection to an RedisPubSub server. 91 func (r *RedisPubSub) ConnectWithContext(ctx context.Context) error { 92 return r.Connect() 93 } 94 95 // Connect establishes a connection to an RedisPubSub server. 96 func (r *RedisPubSub) Connect() error { 97 r.connMut.Lock() 98 defer r.connMut.Unlock() 99 100 client, err := r.conf.Config.Client() 101 if err != nil { 102 return err 103 } 104 if _, err = client.Ping().Result(); err != nil { 105 return err 106 } 107 108 r.log.Infof("Pushing messages to Redis channel: %v\n", r.conf.Channel) 109 110 r.client = client 111 return nil 112 } 113 114 //------------------------------------------------------------------------------ 115 116 // WriteWithContext attempts to write a message by pushing it to a Redis pub/sub 117 // topic. 118 func (r *RedisPubSub) WriteWithContext(ctx context.Context, msg types.Message) error { 119 r.connMut.RLock() 120 client := r.client 121 r.connMut.RUnlock() 122 123 if client == nil { 124 return types.ErrNotConnected 125 } 126 127 if msg.Len() == 1 { 128 channel := r.channelStr.String(0, msg) 129 if err := client.Publish(channel, msg.Get(0).Get()).Err(); err != nil { 130 r.disconnect() 131 r.log.Errorf("Error from redis: %v\n", err) 132 return types.ErrNotConnected 133 } 134 return nil 135 } 136 137 pipe := client.Pipeline() 138 msg.Iter(func(i int, p types.Part) error { 139 _ = pipe.Publish(r.channelStr.String(i, msg), p.Get()) 140 return nil 141 }) 142 cmders, err := pipe.Exec() 143 if err != nil { 144 r.disconnect() 145 r.log.Errorf("Error from redis: %v\n", err) 146 return types.ErrNotConnected 147 } 148 149 var batchErr *ibatch.Error 150 for i, res := range cmders { 151 if res.Err() != nil { 152 if batchErr == nil { 153 batchErr = ibatch.NewError(msg, res.Err()) 154 } 155 batchErr.Failed(i, res.Err()) 156 } 157 } 158 if batchErr != nil { 159 return batchErr 160 } 161 return nil 162 } 163 164 // Write attempts to write a message by pushing it to a Redis pub/sub topic. 165 func (r *RedisPubSub) Write(msg types.Message) error { 166 return r.WriteWithContext(context.Background(), msg) 167 } 168 169 // disconnect safely closes a connection to an RedisPubSub server. 170 func (r *RedisPubSub) disconnect() error { 171 r.connMut.Lock() 172 defer r.connMut.Unlock() 173 if r.client != nil { 174 err := r.client.Close() 175 r.client = nil 176 return err 177 } 178 return nil 179 } 180 181 // CloseAsync shuts down the RedisPubSub output and stops processing messages. 182 func (r *RedisPubSub) CloseAsync() { 183 r.disconnect() 184 } 185 186 // WaitForClose blocks until the RedisPubSub output has closed down. 187 func (r *RedisPubSub) WaitForClose(timeout time.Duration) error { 188 return nil 189 } 190 191 //------------------------------------------------------------------------------