github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/bloblang.go (about) 1 package processor 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/Jeffail/benthos/v3/internal/bloblang/mapping" 8 "github.com/Jeffail/benthos/v3/internal/bloblang/parser" 9 "github.com/Jeffail/benthos/v3/internal/docs" 10 "github.com/Jeffail/benthos/v3/internal/interop" 11 "github.com/Jeffail/benthos/v3/internal/tracing" 12 "github.com/Jeffail/benthos/v3/lib/log" 13 "github.com/Jeffail/benthos/v3/lib/message" 14 "github.com/Jeffail/benthos/v3/lib/metrics" 15 "github.com/Jeffail/benthos/v3/lib/response" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 ) 18 19 //------------------------------------------------------------------------------ 20 21 func init() { 22 Constructors[TypeBloblang] = TypeSpec{ 23 constructor: NewBloblang, 24 Categories: []Category{ 25 CategoryMapping, 26 CategoryParsing, 27 }, 28 config: docs.FieldComponent().HasType(docs.FieldTypeString).IsBloblang().HasDefault(""), 29 Summary: ` 30 Executes a [Bloblang](/docs/guides/bloblang/about) mapping on messages.`, 31 Description: ` 32 Bloblang is a powerful language that enables a wide range of mapping, transformation and filtering tasks. For more information [check out the docs](/docs/guides/bloblang/about). 33 34 If your mapping is large and you'd prefer for it to live in a separate file then you can execute a mapping directly from a file with the expression ` + "`from \"<path>\"`" + `, where the path must be absolute, or relative from the location that Benthos is executed from.`, 35 Footnotes: ` 36 ## Error Handling 37 38 Bloblang mappings can fail, in which case the message remains unchanged, errors 39 are logged, and the message is flagged as having failed, allowing you to use 40 [standard processor error handling patterns](/docs/configuration/error_handling). 41 42 However, Bloblang itself also provides powerful ways of ensuring your mappings 43 do not fail by specifying desired fallback behaviour, which you can read about 44 [in this section](/docs/guides/bloblang/about#error-handling).`, 45 Examples: []docs.AnnotatedExample{ 46 { 47 Title: "Mapping", 48 Summary: ` 49 Given JSON documents containing an array of fans: 50 51 ` + "```json" + ` 52 { 53 "id":"foo", 54 "description":"a show about foo", 55 "fans":[ 56 {"name":"bev","obsession":0.57}, 57 {"name":"grace","obsession":0.21}, 58 {"name":"ali","obsession":0.89}, 59 {"name":"vic","obsession":0.43} 60 ] 61 } 62 ` + "```" + ` 63 64 We can reduce the fans to only those with an obsession score above 0.5, giving us: 65 66 ` + "```json" + ` 67 { 68 "id":"foo", 69 "description":"a show about foo", 70 "fans":[ 71 {"name":"bev","obsession":0.57}, 72 {"name":"ali","obsession":0.89} 73 ] 74 } 75 ` + "```" + ` 76 77 With the following config:`, 78 Config: ` 79 pipeline: 80 processors: 81 - bloblang: | 82 root = this 83 root.fans = this.fans.filter(fan -> fan.obsession > 0.5) 84 `, 85 }, 86 { 87 Title: "More Mapping", 88 Summary: ` 89 When receiving JSON documents of the form: 90 91 ` + "```json" + ` 92 { 93 "locations": [ 94 {"name": "Seattle", "state": "WA"}, 95 {"name": "New York", "state": "NY"}, 96 {"name": "Bellevue", "state": "WA"}, 97 {"name": "Olympia", "state": "WA"} 98 ] 99 } 100 ` + "```" + ` 101 102 We could collapse the location names from the state of Washington into a field ` + "`Cities`" + `: 103 104 ` + "```json" + ` 105 {"Cities": "Bellevue, Olympia, Seattle"} 106 ` + "```" + ` 107 108 With the following config:`, 109 Config: ` 110 pipeline: 111 processors: 112 - bloblang: | 113 root.Cities = this.locations. 114 filter(loc -> loc.state == "WA"). 115 map_each(loc -> loc.name). 116 sort().join(", ") 117 `, 118 }, 119 }, 120 } 121 } 122 123 //------------------------------------------------------------------------------ 124 125 // BloblangConfig contains configuration fields for the Bloblang processor. 126 type BloblangConfig string 127 128 // NewBloblangConfig returns a BloblangConfig with default values. 129 func NewBloblangConfig() BloblangConfig { 130 return "" 131 } 132 133 //------------------------------------------------------------------------------ 134 135 // Bloblang is a processor that performs a Bloblang mapping. 136 type Bloblang struct { 137 exec *mapping.Executor 138 139 log log.Modular 140 stats metrics.Type 141 142 mCount metrics.StatCounter 143 mErr metrics.StatCounter 144 mSent metrics.StatCounter 145 mBatchSent metrics.StatCounter 146 mDropped metrics.StatCounter 147 } 148 149 // NewBloblang returns a Bloblang processor. 150 func NewBloblang( 151 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 152 ) (Type, error) { 153 exec, err := interop.NewBloblangMapping(mgr, string(conf.Bloblang)) 154 if err != nil { 155 if perr, ok := err.(*parser.Error); ok { 156 return nil, fmt.Errorf("%v", perr.ErrorAtPosition([]rune(conf.Bloblang))) 157 } 158 return nil, err 159 } 160 return NewBloblangFromExecutor(exec, log, stats), nil 161 } 162 163 // NewBloblangFromExecutor returns a Bloblang processor. 164 func NewBloblangFromExecutor(exec *mapping.Executor, log log.Modular, stats metrics.Type) Type { 165 return &Bloblang{ 166 exec: exec, 167 168 log: log, 169 stats: stats, 170 171 mCount: stats.GetCounter("count"), 172 mErr: stats.GetCounter("error"), 173 mSent: stats.GetCounter("sent"), 174 mBatchSent: stats.GetCounter("batch.sent"), 175 mDropped: stats.GetCounter("dropped"), 176 } 177 } 178 179 //------------------------------------------------------------------------------ 180 181 // ProcessMessage applies the processor to a message, either creating >0 182 // resulting messages or a response to be sent back to the message source. 183 func (b *Bloblang) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 184 b.mCount.Incr(1) 185 186 newParts := make([]types.Part, 0, msg.Len()) 187 188 msg.Iter(func(i int, part types.Part) error { 189 span := tracing.CreateChildSpan(TypeBloblang, part) 190 191 p, err := b.exec.MapPart(i, msg) 192 if err != nil { 193 p = part.Copy() 194 b.mErr.Incr(1) 195 b.log.Errorf("%v\n", err) 196 FlagErr(p, err) 197 span.SetTag("error", true) 198 span.LogKV( 199 "event", "error", 200 "type", err.Error(), 201 ) 202 } 203 204 span.Finish() 205 if p != nil { 206 newParts = append(newParts, p) 207 } else { 208 b.mDropped.Incr(1) 209 } 210 return nil 211 }) 212 213 if len(newParts) == 0 { 214 return nil, response.NewAck() 215 } 216 217 newMsg := message.New(nil) 218 newMsg.SetAll(newParts) 219 220 b.mBatchSent.Incr(1) 221 b.mSent.Incr(int64(newMsg.Len())) 222 return []types.Message{newMsg}, nil 223 } 224 225 // CloseAsync shuts down the processor and stops processing requests. 226 func (b *Bloblang) CloseAsync() { 227 } 228 229 // WaitForClose blocks until the processor has closed down. 230 func (b *Bloblang) WaitForClose(timeout time.Duration) error { 231 return nil 232 } 233 234 //------------------------------------------------------------------------------