github.com/Jeffail/benthos/v3@v3.65.0/lib/output/writer/http_client.go (about) 1 package writer 2 3 import ( 4 "context" 5 "fmt" 6 "time" 7 8 "github.com/Jeffail/benthos/v3/internal/http" 9 "github.com/Jeffail/benthos/v3/internal/interop" 10 "github.com/Jeffail/benthos/v3/lib/log" 11 "github.com/Jeffail/benthos/v3/lib/message/batch" 12 "github.com/Jeffail/benthos/v3/lib/message/roundtrip" 13 "github.com/Jeffail/benthos/v3/lib/metrics" 14 "github.com/Jeffail/benthos/v3/lib/types" 15 "github.com/Jeffail/benthos/v3/lib/util/http/client" 16 ) 17 18 //------------------------------------------------------------------------------ 19 20 // HTTPClientMultipartExpression represents dynamic expressions that define a 21 // multipart message part in an HTTP request. Specifying one or more of these 22 // can be used as a way of creating HTTP requests that overrides the default 23 // behaviour. 24 type HTTPClientMultipartExpression struct { 25 ContentDisposition string `json:"content_disposition" yaml:"content_disposition"` 26 ContentType string `json:"content_type" yaml:"content_type"` 27 Body string `json:"body" yaml:"body"` 28 } 29 30 // HTTPClientConfig contains configuration fields for the HTTPClient output 31 // type. 32 type HTTPClientConfig struct { 33 client.Config `json:",inline" yaml:",inline"` 34 BatchAsMultipart bool `json:"batch_as_multipart" yaml:"batch_as_multipart"` 35 MaxInFlight int `json:"max_in_flight" yaml:"max_in_flight"` 36 PropagateResponse bool `json:"propagate_response" yaml:"propagate_response"` 37 Batching batch.PolicyConfig `json:"batching" yaml:"batching"` 38 Multipart []HTTPClientMultipartExpression `json:"multipart" yaml:"multipart"` 39 } 40 41 // NewHTTPClientConfig creates a new HTTPClientConfig with default values. 42 func NewHTTPClientConfig() HTTPClientConfig { 43 return HTTPClientConfig{ 44 Config: client.NewConfig(), 45 BatchAsMultipart: true, // TODO: V4 Set false by default. 46 MaxInFlight: 1, // TODO: Increase this default? 47 PropagateResponse: false, 48 Batching: batch.NewPolicyConfig(), 49 } 50 } 51 52 //------------------------------------------------------------------------------ 53 54 // HTTPClient is an output type that sends messages as HTTP requests to a target 55 // server endpoint. 56 type HTTPClient struct { 57 client *http.Client 58 59 stats metrics.Type 60 log log.Modular 61 62 conf HTTPClientConfig 63 closeChan chan struct{} 64 } 65 66 // NewHTTPClient creates a new HTTPClient writer type. 67 func NewHTTPClient( 68 conf HTTPClientConfig, 69 mgr types.Manager, 70 log log.Modular, 71 stats metrics.Type, 72 ) (*HTTPClient, error) { 73 h := HTTPClient{ 74 stats: stats, 75 log: log, 76 conf: conf, 77 closeChan: make(chan struct{}), 78 } 79 80 opts := []func(*http.Client){ 81 http.OptSetLogger(h.log), 82 http.OptSetManager(mgr), 83 // TODO: V4 Remove this 84 http.OptSetStats(metrics.Namespaced(h.stats, "client")), 85 } 86 87 if len(conf.Multipart) > 0 { 88 parts := make([]http.MultipartExpressions, len(conf.Multipart)) 89 for i, p := range conf.Multipart { 90 var exprPart http.MultipartExpressions 91 var err error 92 if exprPart.ContentDisposition, err = interop.NewBloblangField(mgr, p.ContentDisposition); err != nil { 93 return nil, fmt.Errorf("failed to parse multipart %v field content_disposition: %v", i, err) 94 } 95 if exprPart.ContentType, err = interop.NewBloblangField(mgr, p.ContentType); err != nil { 96 return nil, fmt.Errorf("failed to parse multipart %v field content_type: %v", i, err) 97 } 98 if exprPart.Body, err = interop.NewBloblangField(mgr, p.Body); err != nil { 99 return nil, fmt.Errorf("failed to parse multipart %v field data: %v", i, err) 100 } 101 parts[i] = exprPart 102 } 103 opts = append(opts, http.OptSetMultiPart(parts)) 104 } 105 106 var err error 107 if h.client, err = http.NewClient(conf.Config, opts...); err != nil { 108 return nil, err 109 } 110 return &h, nil 111 } 112 113 //------------------------------------------------------------------------------ 114 115 // ConnectWithContext does nothing. 116 func (h *HTTPClient) ConnectWithContext(ctx context.Context) error { 117 h.log.Infof("Sending messages via HTTP requests to: %s\n", h.conf.URL) 118 return nil 119 } 120 121 // Connect does nothing. 122 func (h *HTTPClient) Connect() error { 123 return h.ConnectWithContext(context.Background()) 124 } 125 126 // Write attempts to send a message to an HTTP server, this attempt may include 127 // retries, and if all retries fail an error is returned. 128 func (h *HTTPClient) Write(msg types.Message) error { 129 return h.WriteWithContext(context.Background(), msg) 130 } 131 132 // WriteWithContext attempts to send a message to an HTTP server, this attempt 133 // may include retries, and if all retries fail an error is returned. 134 func (h *HTTPClient) WriteWithContext(ctx context.Context, msg types.Message) error { 135 resultMsg, err := h.client.Send(ctx, msg, msg) 136 if err == nil && h.conf.PropagateResponse { 137 msgCopy := msg.Copy() 138 parts := make([]types.Part, resultMsg.Len()) 139 resultMsg.Iter(func(i int, p types.Part) error { 140 if i < msgCopy.Len() { 141 parts[i] = msgCopy.Get(i) 142 } else { 143 parts[i] = msgCopy.Get(0) 144 } 145 parts[i].Set(p.Get()) 146 147 p.Metadata().Iter(func(k, v string) error { 148 parts[i].Metadata().Set(k, v) 149 return nil 150 }) 151 152 return nil 153 }) 154 msgCopy.SetAll(parts) 155 roundtrip.SetAsResponse(msgCopy) 156 } 157 return err 158 } 159 160 // CloseAsync shuts down the HTTPClient output and stops processing messages. 161 func (h *HTTPClient) CloseAsync() { 162 close(h.closeChan) 163 go h.client.Close(context.Background()) 164 } 165 166 // WaitForClose blocks until the HTTPClient output has closed down. 167 func (h *HTTPClient) WaitForClose(timeout time.Duration) error { 168 return nil 169 } 170 171 //------------------------------------------------------------------------------