github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/http.go (about) 1 package processor 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "strconv" 8 "time" 9 10 "github.com/Jeffail/benthos/v3/internal/docs" 11 "github.com/Jeffail/benthos/v3/internal/http" 12 "github.com/Jeffail/benthos/v3/lib/log" 13 "github.com/Jeffail/benthos/v3/lib/message" 14 "github.com/Jeffail/benthos/v3/lib/metrics" 15 "github.com/Jeffail/benthos/v3/lib/response" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 "github.com/Jeffail/benthos/v3/lib/util/http/auth" 18 "github.com/Jeffail/benthos/v3/lib/util/http/client" 19 "github.com/google/go-cmp/cmp" 20 "github.com/google/go-cmp/cmp/cmpopts" 21 yaml "gopkg.in/yaml.v3" 22 ) 23 24 //------------------------------------------------------------------------------ 25 26 func init() { 27 Constructors[TypeHTTP] = TypeSpec{ 28 constructor: NewHTTP, 29 Categories: []Category{ 30 CategoryIntegration, 31 }, 32 Summary: ` 33 Performs an HTTP request using a message batch as the request body, and replaces 34 the original message parts with the body of the response.`, 35 Description: ` 36 If a processed message batch contains more than one message they will be sent in 37 a single request as a [multipart message](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). 38 Alternatively, message batches can be sent in parallel by setting the field 39 ` + "`parallel` to `true`" + `. 40 41 The ` + "`rate_limit`" + ` field can be used to specify a rate limit 42 [resource](/docs/components/rate_limits/about) to cap the rate of requests 43 across all parallel components service wide. 44 45 The URL and header values of this type can be dynamically set using function 46 interpolations described [here](/docs/configuration/interpolation#bloblang-queries). 47 48 In order to map or encode the payload to a specific request body, and map the 49 response back into the original payload instead of replacing it entirely, you 50 can use the ` + "[`branch` processor](/docs/components/processors/branch)" + `. 51 52 ## Response Codes 53 54 Benthos considers any response code between 200 and 299 inclusive to indicate a 55 successful response, you can add more success status codes with the field 56 ` + "`successful_on`" + `. 57 58 When a request returns a response code within the ` + "`backoff_on`" + ` field 59 it will be retried after increasing intervals. 60 61 When a request returns a response code within the ` + "`drop_on`" + ` field it 62 will not be reattempted and is immediately considered a failed request. 63 64 ## Adding Metadata 65 66 If the request returns an error response code this processor sets a metadata 67 field ` + "`http_status_code`" + ` on the resulting message. 68 69 Use the field ` + "`extract_headers`" + ` to specify rules for which other 70 headers should be copied into the resulting message from the response. 71 72 ## Error Handling 73 74 When all retry attempts for a message are exhausted the processor cancels the 75 attempt. These failed messages will continue through the pipeline unchanged, but 76 can be dropped or placed in a dead letter queue according to your config, you 77 can read about these patterns [here](/docs/configuration/error_handling).`, 78 config: client.FieldSpec( 79 docs.FieldCommon("parallel", "When processing batched messages, whether to send messages of the batch in parallel, otherwise they are sent within a single request."), 80 docs.FieldDeprecated("max_parallel"), 81 docs.FieldDeprecated("request").OmitWhen(func(v, _ interface{}) (string, bool) { 82 defaultBytes, err := yaml.Marshal(client.NewConfig()) 83 if err != nil { 84 return "", false 85 } 86 var iDefault interface{} 87 if err = yaml.Unmarshal(defaultBytes, &iDefault); err != nil { 88 return "", false 89 } 90 return "field request is deprecated", cmp.Equal(v, iDefault) 91 }), 92 ), 93 Examples: []docs.AnnotatedExample{ 94 { 95 Title: "Branched Request", 96 Summary: ` 97 This example uses a ` + "[`branch` processor](/docs/components/processors/branch/)" + ` to strip the request message into an empty body, grab an HTTP payload, and place the result back into the original message at the path ` + "`repo.status`" + `:`, 98 Config: ` 99 pipeline: 100 processors: 101 - branch: 102 request_map: 'root = ""' 103 processors: 104 - http: 105 url: https://hub.docker.com/v2/repositories/jeffail/benthos 106 verb: GET 107 result_map: 'root.repo.status = this' 108 `, 109 }, 110 }, 111 } 112 } 113 114 //------------------------------------------------------------------------------ 115 116 // HTTPConfig contains configuration fields for the HTTP processor. 117 type HTTPConfig struct { 118 Parallel bool `json:"parallel" yaml:"parallel"` 119 MaxParallel int `json:"max_parallel" yaml:"max_parallel"` 120 Client client.Config `json:"request" yaml:"request"` 121 client.Config `json:",inline" yaml:",inline"` 122 } 123 124 // NewHTTPConfig returns a HTTPConfig with default values. 125 func NewHTTPConfig() HTTPConfig { 126 return HTTPConfig{ 127 Client: client.NewConfig(), 128 Parallel: false, 129 MaxParallel: 0, 130 Config: client.NewConfig(), 131 } 132 } 133 134 //------------------------------------------------------------------------------ 135 136 // HTTP is a processor that performs an HTTP request using the message as the 137 // request body, and returns the response. 138 type HTTP struct { 139 client *http.Client 140 141 parallel bool 142 max int 143 144 conf Config 145 log log.Modular 146 stats metrics.Type 147 148 mCount metrics.StatCounter 149 mErrHTTP metrics.StatCounter 150 mErr metrics.StatCounter 151 mSent metrics.StatCounter 152 mBatchSent metrics.StatCounter 153 } 154 155 // NewHTTP returns a HTTP processor. 156 func NewHTTP( 157 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 158 ) (Type, error) { 159 if !cmp.Equal(conf.HTTP.Client, client.NewConfig(), cmpopts.IgnoreUnexported(auth.JWTConfig{})) { 160 if !cmp.Equal(conf.HTTP.Config, client.NewConfig(), cmpopts.IgnoreUnexported(auth.JWTConfig{})) { 161 return nil, fmt.Errorf("detected a mix of both deprecated http.request and standard http config fields") 162 } 163 log.Warnln("Using deprecated http.request fields. All fields under the path http.request should now be written directly within http.") 164 conf.HTTP.Config = conf.HTTP.Client 165 } 166 g := &HTTP{ 167 conf: conf, 168 log: log, 169 stats: stats, 170 171 parallel: conf.HTTP.Parallel, 172 max: conf.HTTP.MaxParallel, 173 174 mCount: stats.GetCounter("count"), 175 mErrHTTP: stats.GetCounter("error.http"), 176 mErr: stats.GetCounter("error"), 177 mSent: stats.GetCounter("sent"), 178 mBatchSent: stats.GetCounter("batch.sent"), 179 } 180 var err error 181 if g.client, err = http.NewClient( 182 conf.HTTP.Config, 183 http.OptSetLogger(g.log), 184 // TODO: V4 Remove this 185 http.OptSetStats(metrics.Namespaced(g.stats, "client")), 186 http.OptSetManager(mgr), 187 ); err != nil { 188 return nil, err 189 } 190 return g, nil 191 } 192 193 //------------------------------------------------------------------------------ 194 195 // ProcessMessage applies the processor to a message, either creating >0 196 // resulting messages or a response to be sent back to the message source. 197 func (h *HTTP) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 198 h.mCount.Incr(1) 199 var responseMsg types.Message 200 201 if !h.parallel || msg.Len() == 1 { 202 // Easy, just do a single request. 203 resultMsg, err := h.client.Send(context.Background(), msg, msg) 204 if err != nil { 205 var codeStr string 206 var hErr types.ErrUnexpectedHTTPRes 207 if ok := errors.As(err, &hErr); ok { 208 codeStr = strconv.Itoa(hErr.Code) 209 } 210 h.mErr.Incr(1) 211 h.mErrHTTP.Incr(1) 212 h.log.Errorf("HTTP request failed: %v\n", err) 213 responseMsg = msg.Copy() 214 responseMsg.Iter(func(i int, p types.Part) error { 215 if len(codeStr) > 0 { 216 p.Metadata().Set("http_status_code", codeStr) 217 } 218 FlagErr(p, err) 219 return nil 220 }) 221 } else { 222 parts := make([]types.Part, resultMsg.Len()) 223 resultMsg.Iter(func(i int, p types.Part) error { 224 if i < msg.Len() { 225 parts[i] = msg.Get(i).Copy() 226 } else { 227 parts[i] = msg.Get(0).Copy() 228 } 229 parts[i].Set(p.Get()) 230 p.Metadata().Iter(func(k, v string) error { 231 parts[i].Metadata().Set(k, v) 232 return nil 233 }) 234 return nil 235 }) 236 responseMsg = message.New(nil) 237 responseMsg.Append(parts...) 238 } 239 } else { 240 // Hard, need to do parallel requests limited by max parallelism. 241 results := make([]types.Part, msg.Len()) 242 msg.Iter(func(i int, p types.Part) error { 243 results[i] = p.Copy() 244 return nil 245 }) 246 reqChan, resChan := make(chan int), make(chan error) 247 248 max := h.max 249 if max == 0 || msg.Len() < max { 250 max = msg.Len() 251 } 252 253 for i := 0; i < max; i++ { 254 go func() { 255 for index := range reqChan { 256 tmpMsg := message.Lock(msg, index) 257 result, err := h.client.Send(context.Background(), tmpMsg, tmpMsg) 258 if err == nil && result.Len() != 1 { 259 err = fmt.Errorf("unexpected response size: %v", result.Len()) 260 } 261 if err == nil { 262 results[index].Set(result.Get(0).Get()) 263 result.Get(0).Metadata().Iter(func(k, v string) error { 264 results[index].Metadata().Set(k, v) 265 return nil 266 }) 267 } else { 268 var hErr types.ErrUnexpectedHTTPRes 269 if ok := errors.As(err, &hErr); ok { 270 results[index].Metadata().Set("http_status_code", strconv.Itoa(hErr.Code)) 271 } 272 FlagErr(results[index], err) 273 } 274 resChan <- err 275 } 276 }() 277 } 278 go func() { 279 for i := 0; i < msg.Len(); i++ { 280 reqChan <- i 281 } 282 }() 283 for i := 0; i < msg.Len(); i++ { 284 if err := <-resChan; err != nil { 285 h.mErr.Incr(1) 286 h.mErrHTTP.Incr(1) 287 h.log.Errorf("HTTP parallel request to '%v' failed: %v\n", h.conf.HTTP.URL, err) 288 } 289 } 290 291 close(reqChan) 292 responseMsg = message.New(nil) 293 responseMsg.Append(results...) 294 } 295 296 if responseMsg.Len() < 1 { 297 return nil, response.NewError(fmt.Errorf( 298 "HTTP response from '%v' was empty", h.conf.HTTP.URL, 299 )) 300 } 301 302 msgs := [1]types.Message{responseMsg} 303 304 h.mBatchSent.Incr(1) 305 h.mSent.Incr(int64(responseMsg.Len())) 306 return msgs[:], nil 307 } 308 309 // CloseAsync shuts down the processor and stops processing requests. 310 func (h *HTTP) CloseAsync() { 311 go h.client.Close(context.Background()) 312 } 313 314 // WaitForClose blocks until the processor has closed down. 315 func (h *HTTP) WaitForClose(timeout time.Duration) error { 316 return nil 317 } 318 319 //------------------------------------------------------------------------------