github.com/Jeffail/benthos/v3@v3.65.0/lib/output/inproc.go (about) 1 package output 2 3 import ( 4 "sync/atomic" 5 "time" 6 7 "github.com/Jeffail/benthos/v3/internal/batch" 8 "github.com/Jeffail/benthos/v3/internal/docs" 9 "github.com/Jeffail/benthos/v3/lib/log" 10 "github.com/Jeffail/benthos/v3/lib/metrics" 11 "github.com/Jeffail/benthos/v3/lib/types" 12 ) 13 14 //------------------------------------------------------------------------------ 15 16 func init() { 17 Constructors[TypeInproc] = TypeSpec{ 18 constructor: fromSimpleConstructor(NewInproc), 19 Description: ` 20 Sends data directly to Benthos inputs by connecting to a unique ID. This allows 21 you to hook up isolated streams whilst running Benthos in 22 ` + "[streams mode](/docs/guides/streams_mode/about)" + `, it is NOT recommended 23 that you connect the inputs of a stream with an output of the same stream, as 24 feedback loops can lead to deadlocks in your message flow. 25 26 It is possible to connect multiple inputs to the same inproc ID, resulting in 27 messages dispatching in a round-robin fashion to connected inputs. However, only 28 one output can assume an inproc ID, and will replace existing outputs if a 29 collision occurs.`, 30 Categories: []Category{ 31 CategoryUtility, 32 }, 33 config: docs.FieldComponent().HasType(docs.FieldTypeString).HasDefault(""), 34 } 35 } 36 37 //------------------------------------------------------------------------------ 38 39 // InprocConfig contains configuration fields for the Inproc output type. 40 type InprocConfig string 41 42 // NewInprocConfig creates a new InprocConfig with default values. 43 func NewInprocConfig() InprocConfig { 44 return InprocConfig("") 45 } 46 47 //------------------------------------------------------------------------------ 48 49 // Inproc is an output type that serves Inproc messages. 50 type Inproc struct { 51 running int32 52 53 pipe string 54 mgr types.Manager 55 log log.Modular 56 stats metrics.Type 57 58 transactionsOut chan types.Transaction 59 transactionsIn <-chan types.Transaction 60 61 closedChan chan struct{} 62 closeChan chan struct{} 63 } 64 65 // NewInproc creates a new Inproc output type. 66 func NewInproc(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) { 67 i := &Inproc{ 68 running: 1, 69 pipe: string(conf.Inproc), 70 mgr: mgr, 71 log: log, 72 stats: stats, 73 transactionsOut: make(chan types.Transaction), 74 closedChan: make(chan struct{}), 75 closeChan: make(chan struct{}), 76 } 77 mgr.SetPipe(i.pipe, i.transactionsOut) 78 return i, nil 79 } 80 81 //------------------------------------------------------------------------------ 82 83 // loop is an internal loop that brokers incoming messages to output pipe. 84 func (i *Inproc) loop() { 85 var ( 86 mRunning = i.stats.GetGauge("running") 87 mCount = i.stats.GetCounter("count") 88 mPartsCount = i.stats.GetCounter("parts.count") 89 mSendSucc = i.stats.GetCounter("send.success") 90 mPartsSendSucc = i.stats.GetCounter("parts.send.success") 91 mSent = i.stats.GetCounter("batch.sent") 92 mPartsSent = i.stats.GetCounter("sent") 93 ) 94 95 defer func() { 96 mRunning.Decr(1) 97 atomic.StoreInt32(&i.running, 0) 98 i.mgr.UnsetPipe(i.pipe, i.transactionsOut) 99 close(i.transactionsOut) 100 close(i.closedChan) 101 }() 102 mRunning.Incr(1) 103 104 i.log.Infof("Sending inproc messages to ID: %s\n", i.pipe) 105 106 var open bool 107 for atomic.LoadInt32(&i.running) == 1 { 108 var ts types.Transaction 109 select { 110 case ts, open = <-i.transactionsIn: 111 if !open { 112 return 113 } 114 case <-i.closeChan: 115 return 116 } 117 118 mCount.Incr(1) 119 if ts.Payload != nil { 120 mPartsCount.Incr(int64(ts.Payload.Len())) 121 } 122 select { 123 case i.transactionsOut <- ts: 124 mSendSucc.Incr(1) 125 mSent.Incr(1) 126 if ts.Payload != nil { 127 mPartsSendSucc.Incr(int64(ts.Payload.Len())) 128 mPartsSent.Incr(int64(batch.MessageCollapsedCount(ts.Payload))) 129 } 130 case <-i.closeChan: 131 return 132 } 133 } 134 } 135 136 // Consume assigns a messages channel for the output to read. 137 func (i *Inproc) Consume(ts <-chan types.Transaction) error { 138 if i.transactionsIn != nil { 139 return types.ErrAlreadyStarted 140 } 141 i.transactionsIn = ts 142 go i.loop() 143 return nil 144 } 145 146 // Connected returns a boolean indicating whether this output is currently 147 // connected to its target. 148 func (i *Inproc) Connected() bool { 149 return true 150 } 151 152 // CloseAsync shuts down the Inproc output and stops processing messages. 153 func (i *Inproc) CloseAsync() { 154 if atomic.CompareAndSwapInt32(&i.running, 1, 0) { 155 close(i.closeChan) 156 } 157 } 158 159 // WaitForClose blocks until the Inproc output has closed down. 160 func (i *Inproc) WaitForClose(timeout time.Duration) error { 161 select { 162 case <-i.closedChan: 163 case <-time.After(timeout): 164 return types.ErrTimeout 165 } 166 return nil 167 } 168 169 //------------------------------------------------------------------------------