github.com/Jeffail/benthos/v3@v3.65.0/lib/input/async_reader.go (about) 1 package input 2 3 import ( 4 "context" 5 "sync" 6 "sync/atomic" 7 "time" 8 9 "github.com/Jeffail/benthos/v3/internal/shutdown" 10 "github.com/Jeffail/benthos/v3/internal/tracing" 11 "github.com/Jeffail/benthos/v3/lib/input/reader" 12 "github.com/Jeffail/benthos/v3/lib/log" 13 "github.com/Jeffail/benthos/v3/lib/metrics" 14 "github.com/Jeffail/benthos/v3/lib/response" 15 "github.com/Jeffail/benthos/v3/lib/types" 16 "github.com/cenkalti/backoff/v4" 17 ) 18 19 //------------------------------------------------------------------------------ 20 21 // AsyncReader is an input implementation that reads messages from a 22 // reader.Async component. 23 type AsyncReader struct { 24 connected int32 25 connBackoff backoff.BackOff 26 27 allowSkipAcks bool 28 29 typeStr string 30 reader reader.Async 31 32 stats metrics.Type 33 log log.Modular 34 35 transactions chan types.Transaction 36 shutSig *shutdown.Signaller 37 } 38 39 // NewAsyncReader creates a new AsyncReader input type. 40 func NewAsyncReader( 41 typeStr string, 42 allowSkipAcks bool, 43 r reader.Async, 44 log log.Modular, 45 stats metrics.Type, 46 ) (Type, error) { 47 boff := backoff.NewExponentialBackOff() 48 boff.InitialInterval = time.Millisecond * 100 49 boff.MaxInterval = time.Second 50 boff.MaxElapsedTime = 0 51 52 rdr := &AsyncReader{ 53 connBackoff: boff, 54 allowSkipAcks: allowSkipAcks, 55 typeStr: typeStr, 56 reader: r, 57 log: log, 58 stats: stats, 59 transactions: make(chan types.Transaction), 60 shutSig: shutdown.NewSignaller(), 61 } 62 63 go rdr.loop() 64 return rdr, nil 65 } 66 67 //------------------------------------------------------------------------------ 68 69 func (r *AsyncReader) loop() { 70 // Metrics paths 71 var ( 72 mRunning = r.stats.GetGauge("running") 73 mCount = r.stats.GetCounter("count") 74 mRcvd = r.stats.GetCounter("batch.received") 75 mPartsRcvd = r.stats.GetCounter("received") 76 mConn = r.stats.GetCounter("connection.up") 77 mFailedConn = r.stats.GetCounter("connection.failed") 78 mLostConn = r.stats.GetCounter("connection.lost") 79 mLatency = r.stats.GetTimer("latency") 80 ) 81 82 defer func() { 83 r.reader.CloseAsync() 84 go func() { 85 select { 86 case <-r.shutSig.CloseNowChan(): 87 _ = r.reader.WaitForClose(0) 88 case <-r.shutSig.HasClosedChan(): 89 } 90 }() 91 _ = r.reader.WaitForClose(shutdown.MaximumShutdownWait()) 92 93 mRunning.Decr(1) 94 atomic.StoreInt32(&r.connected, 0) 95 96 close(r.transactions) 97 r.shutSig.ShutdownComplete() 98 }() 99 mRunning.Incr(1) 100 101 pendingAcks := sync.WaitGroup{} 102 defer func() { 103 r.log.Debugln("Waiting for pending acks to resolve before shutting down.") 104 pendingAcks.Wait() 105 r.log.Debugln("Pending acks resolved.") 106 }() 107 108 initConnection := func() bool { 109 initConnCtx, initConnDone := r.shutSig.CloseAtLeisureCtx(context.Background()) 110 defer initConnDone() 111 for { 112 if err := r.reader.ConnectWithContext(initConnCtx); err != nil { 113 if r.shutSig.ShouldCloseAtLeisure() || err == types.ErrTypeClosed { 114 return false 115 } 116 r.log.Errorf("Failed to connect to %v: %v\n", r.typeStr, err) 117 mFailedConn.Incr(1) 118 select { 119 case <-time.After(r.connBackoff.NextBackOff()): 120 case <-initConnCtx.Done(): 121 return false 122 } 123 } else { 124 r.connBackoff.Reset() 125 return true 126 } 127 } 128 } 129 if !initConnection() { 130 return 131 } 132 mConn.Incr(1) 133 atomic.StoreInt32(&r.connected, 1) 134 135 for { 136 readCtx, readDone := r.shutSig.CloseAtLeisureCtx(context.Background()) 137 msg, ackFn, err := r.reader.ReadWithContext(readCtx) 138 readDone() 139 140 // If our reader says it is not connected. 141 if err == types.ErrNotConnected { 142 mLostConn.Incr(1) 143 atomic.StoreInt32(&r.connected, 0) 144 145 // Continue to try to reconnect while still active. 146 if !initConnection() { 147 return 148 } 149 mConn.Incr(1) 150 atomic.StoreInt32(&r.connected, 1) 151 } 152 153 // Close immediately if our reader is closed. 154 if r.shutSig.ShouldCloseAtLeisure() || err == types.ErrTypeClosed { 155 return 156 } 157 158 if err != nil || msg == nil { 159 if err != nil && err != types.ErrTimeout && err != types.ErrNotConnected { 160 r.log.Errorf("Failed to read message: %v\n", err) 161 } 162 select { 163 case <-time.After(r.connBackoff.NextBackOff()): 164 case <-r.shutSig.CloseAtLeisureChan(): 165 return 166 } 167 continue 168 } else { 169 r.connBackoff.Reset() 170 mCount.Incr(1) 171 mPartsRcvd.Incr(int64(msg.Len())) 172 mRcvd.Incr(1) 173 r.log.Tracef("Consumed %v messages from '%v'.\n", msg.Len(), r.typeStr) 174 } 175 176 resChan := make(chan types.Response) 177 tracing.InitSpans("input_"+r.typeStr, msg) 178 select { 179 case r.transactions <- types.NewTransaction(msg, resChan): 180 case <-r.shutSig.CloseAtLeisureChan(): 181 return 182 } 183 184 pendingAcks.Add(1) 185 go func( 186 m types.Message, 187 aFn reader.AsyncAckFn, 188 rChan chan types.Response, 189 ) { 190 defer pendingAcks.Done() 191 192 var res types.Response 193 var open bool 194 select { 195 case res, open = <-rChan: 196 case <-r.shutSig.CloseNowChan(): 197 // Even if the pipeline is terminating we still want to attempt 198 // to propagate an acknowledgement from in-transit messages. 199 return 200 } 201 if !open { 202 return 203 } 204 if res.SkipAck() && !r.allowSkipAcks { 205 r.log.Errorf("Detected downstream batch processor which is not permitted with this input, please refer to the documentation for more information. This input will now shut down.") 206 res = response.NewNoack() 207 r.CloseAsync() 208 } 209 mLatency.Timing(time.Since(m.CreatedAt()).Nanoseconds()) 210 tracing.FinishSpans(m) 211 212 ackCtx, ackDone := r.shutSig.CloseNowCtx(context.Background()) 213 if err = aFn(ackCtx, res); err != nil { 214 r.log.Errorf("Failed to acknowledge message: %v\n", err) 215 } 216 ackDone() 217 }(msg, ackFn, resChan) 218 } 219 } 220 221 // TransactionChan returns a transactions channel for consuming messages from 222 // this input type. 223 func (r *AsyncReader) TransactionChan() <-chan types.Transaction { 224 return r.transactions 225 } 226 227 // Connected returns a boolean indicating whether this input is currently 228 // connected to its target. 229 func (r *AsyncReader) Connected() bool { 230 return atomic.LoadInt32(&r.connected) == 1 231 } 232 233 // CloseAsync shuts down the AsyncReader input and stops processing requests. 234 func (r *AsyncReader) CloseAsync() { 235 r.shutSig.CloseAtLeisure() 236 } 237 238 // WaitForClose blocks until the AsyncReader input has closed down. 239 func (r *AsyncReader) WaitForClose(timeout time.Duration) error { 240 go func() { 241 <-time.After(timeout - time.Second) 242 r.shutSig.CloseNow() 243 }() 244 select { 245 case <-r.shutSig.HasClosedChan(): 246 case <-time.After(timeout): 247 return types.ErrTimeout 248 } 249 return nil 250 } 251 252 //------------------------------------------------------------------------------