github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/lib/rest/rest.go (about) 1 // Package rest implements a simple REST wrapper 2 // 3 // All methods are safe for concurrent calling. 4 package rest 5 6 import ( 7 "bytes" 8 "context" 9 "encoding/json" 10 "encoding/xml" 11 "io" 12 "io/ioutil" 13 "mime/multipart" 14 "net/http" 15 "net/url" 16 "sync" 17 18 "github.com/pkg/errors" 19 "github.com/rclone/rclone/fs" 20 "github.com/rclone/rclone/lib/readers" 21 ) 22 23 // Client contains the info to sustain the API 24 type Client struct { 25 mu sync.RWMutex 26 c *http.Client 27 rootURL string 28 errorHandler func(resp *http.Response) error 29 headers map[string]string 30 signer SignerFn 31 } 32 33 // NewClient takes an oauth http.Client and makes a new api instance 34 func NewClient(c *http.Client) *Client { 35 api := &Client{ 36 c: c, 37 errorHandler: defaultErrorHandler, 38 headers: make(map[string]string), 39 } 40 return api 41 } 42 43 // ReadBody reads resp.Body into result, closing the body 44 func ReadBody(resp *http.Response) (result []byte, err error) { 45 defer fs.CheckClose(resp.Body, &err) 46 return ioutil.ReadAll(resp.Body) 47 } 48 49 // defaultErrorHandler doesn't attempt to parse the http body, just 50 // returns it in the error message closing resp.Body 51 func defaultErrorHandler(resp *http.Response) (err error) { 52 body, err := ReadBody(resp) 53 if err != nil { 54 return errors.Wrap(err, "error reading error out of body") 55 } 56 return errors.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body) 57 } 58 59 // SetErrorHandler sets the handler to decode an error response when 60 // the HTTP status code is not 2xx. The handler should close resp.Body. 61 func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) *Client { 62 api.mu.Lock() 63 defer api.mu.Unlock() 64 api.errorHandler = fn 65 return api 66 } 67 68 // SetRoot sets the default RootURL. You can override this on a per 69 // call basis using the RootURL field in Opts. 70 func (api *Client) SetRoot(RootURL string) *Client { 71 api.mu.Lock() 72 defer api.mu.Unlock() 73 api.rootURL = RootURL 74 return api 75 } 76 77 // SetHeader sets a header for all requests 78 // Start the key with "*" for don't canonicalise 79 func (api *Client) SetHeader(key, value string) *Client { 80 api.mu.Lock() 81 defer api.mu.Unlock() 82 api.headers[key] = value 83 return api 84 } 85 86 // RemoveHeader unsets a header for all requests 87 func (api *Client) RemoveHeader(key string) *Client { 88 api.mu.Lock() 89 defer api.mu.Unlock() 90 delete(api.headers, key) 91 return api 92 } 93 94 // SignerFn is used to sign an outgoing request 95 type SignerFn func(*http.Request) error 96 97 // SetSigner sets a signer for all requests 98 func (api *Client) SetSigner(signer SignerFn) *Client { 99 api.mu.Lock() 100 defer api.mu.Unlock() 101 api.signer = signer 102 return api 103 } 104 105 // SetUserPass creates an Authorization header for all requests with 106 // the UserName and Password passed in 107 func (api *Client) SetUserPass(UserName, Password string) *Client { 108 req, _ := http.NewRequest("GET", "http://example.com", nil) 109 req.SetBasicAuth(UserName, Password) 110 api.SetHeader("Authorization", req.Header.Get("Authorization")) 111 return api 112 } 113 114 // SetCookie creates an Cookies Header for all requests with the supplied 115 // cookies passed in. 116 // All cookies have to be supplied at once, all cookies will be overwritten 117 // on a new call to the method 118 func (api *Client) SetCookie(cks ...*http.Cookie) *Client { 119 req, _ := http.NewRequest("GET", "http://example.com", nil) 120 for _, ck := range cks { 121 req.AddCookie(ck) 122 } 123 api.SetHeader("Cookie", req.Header.Get("Cookie")) 124 return api 125 } 126 127 // Opts contains parameters for Call, CallJSON etc 128 type Opts struct { 129 Method string // GET, POST etc 130 Path string // relative to RootURL 131 RootURL string // override RootURL passed into SetRoot() 132 Body io.Reader 133 NoResponse bool // set to close Body 134 ContentType string 135 ContentLength *int64 136 ContentRange string 137 ExtraHeaders map[string]string // extra headers, start them with "*" for don't canonicalise 138 UserName string // username for Basic Auth 139 Password string // password for Basic Auth 140 Options []fs.OpenOption 141 IgnoreStatus bool // if set then we don't check error status or parse error body 142 MultipartParams url.Values // if set do multipart form upload with attached file 143 MultipartMetadataName string // ..this is used for the name of the metadata form part if set 144 MultipartContentName string // ..name of the parameter which is the attached file 145 MultipartFileName string // ..name of the file for the attached file 146 Parameters url.Values // any parameters for the final URL 147 TransferEncoding []string // transfer encoding, set to "identity" to disable chunked encoding 148 Close bool // set to close the connection after this transaction 149 NoRedirect bool // if this is set then the client won't follow redirects 150 } 151 152 // Copy creates a copy of the options 153 func (o *Opts) Copy() *Opts { 154 newOpts := *o 155 return &newOpts 156 } 157 158 // DecodeJSON decodes resp.Body into result 159 func DecodeJSON(resp *http.Response, result interface{}) (err error) { 160 defer fs.CheckClose(resp.Body, &err) 161 decoder := json.NewDecoder(resp.Body) 162 return decoder.Decode(result) 163 } 164 165 // DecodeXML decodes resp.Body into result 166 func DecodeXML(resp *http.Response, result interface{}) (err error) { 167 defer fs.CheckClose(resp.Body, &err) 168 decoder := xml.NewDecoder(resp.Body) 169 return decoder.Decode(result) 170 } 171 172 // ClientWithNoRedirects makes a new http client which won't follow redirects 173 func ClientWithNoRedirects(c *http.Client) *http.Client { 174 clientCopy := *c 175 clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error { 176 return http.ErrUseLastResponse 177 } 178 return &clientCopy 179 } 180 181 // Call makes the call and returns the http.Response 182 // 183 // if err == nil then resp.Body will need to be closed unless 184 // opt.NoResponse is set 185 // 186 // if err != nil then resp.Body will have been closed 187 // 188 // it will return resp if at all possible, even if err is set 189 func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, err error) { 190 api.mu.RLock() 191 defer api.mu.RUnlock() 192 if opts == nil { 193 return nil, errors.New("call() called with nil opts") 194 } 195 url := api.rootURL 196 if opts.RootURL != "" { 197 url = opts.RootURL 198 } 199 if url == "" { 200 return nil, errors.New("RootURL not set") 201 } 202 url += opts.Path 203 if opts.Parameters != nil && len(opts.Parameters) > 0 { 204 url += "?" + opts.Parameters.Encode() 205 } 206 body := readers.NoCloser(opts.Body) 207 // If length is set and zero then nil out the body to stop use 208 // use of chunked encoding and insert a "Content-Length: 0" 209 // header. 210 // 211 // If we don't do this we get "Content-Length" headers for all 212 // files except 0 length files. 213 if opts.ContentLength != nil && *opts.ContentLength == 0 { 214 body = nil 215 } 216 req, err := http.NewRequest(opts.Method, url, body) 217 if err != nil { 218 return 219 } 220 req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext 221 headers := make(map[string]string) 222 // Set default headers 223 for k, v := range api.headers { 224 headers[k] = v 225 } 226 if opts.ContentType != "" { 227 headers["Content-Type"] = opts.ContentType 228 } 229 if opts.ContentLength != nil { 230 req.ContentLength = *opts.ContentLength 231 } 232 if opts.ContentRange != "" { 233 headers["Content-Range"] = opts.ContentRange 234 } 235 if len(opts.TransferEncoding) != 0 { 236 req.TransferEncoding = opts.TransferEncoding 237 } 238 if opts.Close { 239 req.Close = true 240 } 241 // Set any extra headers 242 if opts.ExtraHeaders != nil { 243 for k, v := range opts.ExtraHeaders { 244 headers[k] = v 245 } 246 } 247 // add any options to the headers 248 fs.OpenOptionAddHeaders(opts.Options, headers) 249 // Now set the headers 250 for k, v := range headers { 251 if k != "" && v != "" { 252 if k[0] == '*' { 253 // Add non-canonical version if header starts with * 254 k = k[1:] 255 req.Header[k] = append(req.Header[k], v) 256 } else { 257 req.Header.Add(k, v) 258 } 259 } 260 } 261 262 if opts.UserName != "" || opts.Password != "" { 263 req.SetBasicAuth(opts.UserName, opts.Password) 264 } 265 var c *http.Client 266 if opts.NoRedirect { 267 c = ClientWithNoRedirects(api.c) 268 } else { 269 c = api.c 270 } 271 if api.signer != nil { 272 api.mu.RUnlock() 273 err = api.signer(req) 274 api.mu.RLock() 275 if err != nil { 276 return nil, errors.Wrap(err, "signer failed") 277 } 278 } 279 api.mu.RUnlock() 280 resp, err = c.Do(req) 281 api.mu.RLock() 282 if err != nil { 283 return nil, err 284 } 285 if !opts.IgnoreStatus { 286 if resp.StatusCode < 200 || resp.StatusCode > 299 { 287 err = api.errorHandler(resp) 288 if err.Error() == "" { 289 // replace empty errors with something 290 err = errors.Errorf("http error %d: %v", resp.StatusCode, resp.Status) 291 } 292 return resp, err 293 } 294 } 295 if opts.NoResponse { 296 return resp, resp.Body.Close() 297 } 298 return resp, nil 299 } 300 301 // MultipartUpload creates an io.Reader which produces an encoded a 302 // multipart form upload from the params passed in and the passed in 303 // 304 // in - the body of the file (may be nil) 305 // params - the form parameters 306 // fileName - is the name of the attached file 307 // contentName - the name of the parameter for the file 308 // 309 // the int64 returned is the overhead in addition to the file contents, in case Content-Length is required 310 // 311 // NB This doesn't allow setting the content type of the attachment 312 func MultipartUpload(in io.Reader, params url.Values, contentName, fileName string) (io.ReadCloser, string, int64, error) { 313 bodyReader, bodyWriter := io.Pipe() 314 writer := multipart.NewWriter(bodyWriter) 315 contentType := writer.FormDataContentType() 316 317 // Create a Multipart Writer as base for calculating the Content-Length 318 buf := &bytes.Buffer{} 319 dummyMultipartWriter := multipart.NewWriter(buf) 320 err := dummyMultipartWriter.SetBoundary(writer.Boundary()) 321 if err != nil { 322 return nil, "", 0, err 323 } 324 325 for key, vals := range params { 326 for _, val := range vals { 327 err := dummyMultipartWriter.WriteField(key, val) 328 if err != nil { 329 return nil, "", 0, err 330 } 331 } 332 } 333 if in != nil { 334 _, err = dummyMultipartWriter.CreateFormFile(contentName, fileName) 335 if err != nil { 336 return nil, "", 0, err 337 } 338 } 339 340 err = dummyMultipartWriter.Close() 341 if err != nil { 342 return nil, "", 0, err 343 } 344 345 multipartLength := int64(buf.Len()) 346 347 // Pump the data in the background 348 go func() { 349 var err error 350 351 for key, vals := range params { 352 for _, val := range vals { 353 err = writer.WriteField(key, val) 354 if err != nil { 355 _ = bodyWriter.CloseWithError(errors.Wrap(err, "create metadata part")) 356 return 357 } 358 } 359 } 360 361 if in != nil { 362 part, err := writer.CreateFormFile(contentName, fileName) 363 if err != nil { 364 _ = bodyWriter.CloseWithError(errors.Wrap(err, "failed to create form file")) 365 return 366 } 367 368 _, err = io.Copy(part, in) 369 if err != nil { 370 _ = bodyWriter.CloseWithError(errors.Wrap(err, "failed to copy data")) 371 return 372 } 373 } 374 375 err = writer.Close() 376 if err != nil { 377 _ = bodyWriter.CloseWithError(errors.Wrap(err, "failed to close form")) 378 return 379 } 380 381 _ = bodyWriter.Close() 382 }() 383 384 return bodyReader, contentType, multipartLength, nil 385 } 386 387 // CallJSON runs Call and decodes the body as a JSON object into response (if not nil) 388 // 389 // If request is not nil then it will be JSON encoded as the body of the request 390 // 391 // If response is not nil then the response will be JSON decoded into 392 // it and resp.Body will be closed. 393 // 394 // If response is nil then the resp.Body will be closed only if 395 // opts.NoResponse is set. 396 // 397 // If (opts.MultipartParams or opts.MultipartContentName) and 398 // opts.Body are set then CallJSON will do a multipart upload with a 399 // file attached. opts.MultipartContentName is the name of the 400 // parameter and opts.MultipartFileName is the name of the file. If 401 // MultpartContentName is set, and request != nil is supplied, then 402 // the request will be marshalled into JSON and added to the form with 403 // parameter name MultipartMetadataName. 404 // 405 // It will return resp if at all possible, even if err is set 406 func (api *Client) CallJSON(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { 407 return api.callCodec(ctx, opts, request, response, json.Marshal, DecodeJSON, "application/json") 408 } 409 410 // CallXML runs Call and decodes the body as a XML object into response (if not nil) 411 // 412 // If request is not nil then it will be XML encoded as the body of the request 413 // 414 // If response is not nil then the response will be XML decoded into 415 // it and resp.Body will be closed. 416 // 417 // If response is nil then the resp.Body will be closed only if 418 // opts.NoResponse is set. 419 // 420 // See CallJSON for a description of MultipartParams and related opts 421 // 422 // It will return resp if at all possible, even if err is set 423 func (api *Client) CallXML(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { 424 return api.callCodec(ctx, opts, request, response, xml.Marshal, DecodeXML, "application/xml") 425 } 426 427 type marshalFn func(v interface{}) ([]byte, error) 428 type decodeFn func(resp *http.Response, result interface{}) (err error) 429 430 func (api *Client) callCodec(ctx context.Context, opts *Opts, request interface{}, response interface{}, marshal marshalFn, decode decodeFn, contentType string) (resp *http.Response, err error) { 431 var requestBody []byte 432 // Marshal the request if given 433 if request != nil { 434 requestBody, err = marshal(request) 435 if err != nil { 436 return nil, err 437 } 438 // Set the body up as a marshalled object if no body passed in 439 if opts.Body == nil { 440 opts = opts.Copy() 441 opts.ContentType = contentType 442 opts.Body = bytes.NewBuffer(requestBody) 443 } 444 } 445 if opts.MultipartParams != nil || opts.MultipartContentName != "" { 446 params := opts.MultipartParams 447 if params == nil { 448 params = url.Values{} 449 } 450 if opts.MultipartMetadataName != "" { 451 params.Add(opts.MultipartMetadataName, string(requestBody)) 452 } 453 opts = opts.Copy() 454 455 var overhead int64 456 opts.Body, opts.ContentType, overhead, err = MultipartUpload(opts.Body, params, opts.MultipartContentName, opts.MultipartFileName) 457 if err != nil { 458 return nil, err 459 } 460 if opts.ContentLength != nil { 461 *opts.ContentLength += overhead 462 } 463 } 464 resp, err = api.Call(ctx, opts) 465 if err != nil { 466 return resp, err 467 } 468 // if opts.NoResponse is set, resp.Body will have been closed by Call() 469 if response == nil || opts.NoResponse { 470 return resp, nil 471 } 472 err = decode(resp, response) 473 return resp, err 474 }