github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/lib/rest/rest.go (about)

     1  // Package rest implements a simple REST wrapper
     2  //
     3  // All methods are safe for concurrent calling.
     4  package rest
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"encoding/json"
    10  	"encoding/xml"
    11  	"io"
    12  	"io/ioutil"
    13  	"mime/multipart"
    14  	"net/http"
    15  	"net/url"
    16  	"sync"
    17  
    18  	"github.com/pkg/errors"
    19  	"github.com/rclone/rclone/fs"
    20  	"github.com/rclone/rclone/lib/readers"
    21  )
    22  
    23  // Client contains the info to sustain the API
    24  type Client struct {
    25  	mu           sync.RWMutex
    26  	c            *http.Client
    27  	rootURL      string
    28  	errorHandler func(resp *http.Response) error
    29  	headers      map[string]string
    30  	signer       SignerFn
    31  }
    32  
    33  // NewClient takes an oauth http.Client and makes a new api instance
    34  func NewClient(c *http.Client) *Client {
    35  	api := &Client{
    36  		c:            c,
    37  		errorHandler: defaultErrorHandler,
    38  		headers:      make(map[string]string),
    39  	}
    40  	return api
    41  }
    42  
    43  // ReadBody reads resp.Body into result, closing the body
    44  func ReadBody(resp *http.Response) (result []byte, err error) {
    45  	defer fs.CheckClose(resp.Body, &err)
    46  	return ioutil.ReadAll(resp.Body)
    47  }
    48  
    49  // defaultErrorHandler doesn't attempt to parse the http body, just
    50  // returns it in the error message closing resp.Body
    51  func defaultErrorHandler(resp *http.Response) (err error) {
    52  	body, err := ReadBody(resp)
    53  	if err != nil {
    54  		return errors.Wrap(err, "error reading error out of body")
    55  	}
    56  	return errors.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body)
    57  }
    58  
    59  // SetErrorHandler sets the handler to decode an error response when
    60  // the HTTP status code is not 2xx.  The handler should close resp.Body.
    61  func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) *Client {
    62  	api.mu.Lock()
    63  	defer api.mu.Unlock()
    64  	api.errorHandler = fn
    65  	return api
    66  }
    67  
    68  // SetRoot sets the default RootURL.  You can override this on a per
    69  // call basis using the RootURL field in Opts.
    70  func (api *Client) SetRoot(RootURL string) *Client {
    71  	api.mu.Lock()
    72  	defer api.mu.Unlock()
    73  	api.rootURL = RootURL
    74  	return api
    75  }
    76  
    77  // SetHeader sets a header for all requests
    78  // Start the key with "*" for don't canonicalise
    79  func (api *Client) SetHeader(key, value string) *Client {
    80  	api.mu.Lock()
    81  	defer api.mu.Unlock()
    82  	api.headers[key] = value
    83  	return api
    84  }
    85  
    86  // RemoveHeader unsets a header for all requests
    87  func (api *Client) RemoveHeader(key string) *Client {
    88  	api.mu.Lock()
    89  	defer api.mu.Unlock()
    90  	delete(api.headers, key)
    91  	return api
    92  }
    93  
    94  // SignerFn is used to sign an outgoing request
    95  type SignerFn func(*http.Request) error
    96  
    97  // SetSigner sets a signer for all requests
    98  func (api *Client) SetSigner(signer SignerFn) *Client {
    99  	api.mu.Lock()
   100  	defer api.mu.Unlock()
   101  	api.signer = signer
   102  	return api
   103  }
   104  
   105  // SetUserPass creates an Authorization header for all requests with
   106  // the UserName and Password passed in
   107  func (api *Client) SetUserPass(UserName, Password string) *Client {
   108  	req, _ := http.NewRequest("GET", "http://example.com", nil)
   109  	req.SetBasicAuth(UserName, Password)
   110  	api.SetHeader("Authorization", req.Header.Get("Authorization"))
   111  	return api
   112  }
   113  
   114  // SetCookie creates an Cookies Header for all requests with the supplied
   115  // cookies passed in.
   116  // All cookies have to be supplied at once, all cookies will be overwritten
   117  // on a new call to the method
   118  func (api *Client) SetCookie(cks ...*http.Cookie) *Client {
   119  	req, _ := http.NewRequest("GET", "http://example.com", nil)
   120  	for _, ck := range cks {
   121  		req.AddCookie(ck)
   122  	}
   123  	api.SetHeader("Cookie", req.Header.Get("Cookie"))
   124  	return api
   125  }
   126  
   127  // Opts contains parameters for Call, CallJSON etc
   128  type Opts struct {
   129  	Method                string // GET, POST etc
   130  	Path                  string // relative to RootURL
   131  	RootURL               string // override RootURL passed into SetRoot()
   132  	Body                  io.Reader
   133  	NoResponse            bool // set to close Body
   134  	ContentType           string
   135  	ContentLength         *int64
   136  	ContentRange          string
   137  	ExtraHeaders          map[string]string // extra headers, start them with "*" for don't canonicalise
   138  	UserName              string            // username for Basic Auth
   139  	Password              string            // password for Basic Auth
   140  	Options               []fs.OpenOption
   141  	IgnoreStatus          bool       // if set then we don't check error status or parse error body
   142  	MultipartParams       url.Values // if set do multipart form upload with attached file
   143  	MultipartMetadataName string     // ..this is used for the name of the metadata form part if set
   144  	MultipartContentName  string     // ..name of the parameter which is the attached file
   145  	MultipartFileName     string     // ..name of the file for the attached file
   146  	Parameters            url.Values // any parameters for the final URL
   147  	TransferEncoding      []string   // transfer encoding, set to "identity" to disable chunked encoding
   148  	Close                 bool       // set to close the connection after this transaction
   149  	NoRedirect            bool       // if this is set then the client won't follow redirects
   150  }
   151  
   152  // Copy creates a copy of the options
   153  func (o *Opts) Copy() *Opts {
   154  	newOpts := *o
   155  	return &newOpts
   156  }
   157  
   158  // DecodeJSON decodes resp.Body into result
   159  func DecodeJSON(resp *http.Response, result interface{}) (err error) {
   160  	defer fs.CheckClose(resp.Body, &err)
   161  	decoder := json.NewDecoder(resp.Body)
   162  	return decoder.Decode(result)
   163  }
   164  
   165  // DecodeXML decodes resp.Body into result
   166  func DecodeXML(resp *http.Response, result interface{}) (err error) {
   167  	defer fs.CheckClose(resp.Body, &err)
   168  	decoder := xml.NewDecoder(resp.Body)
   169  	return decoder.Decode(result)
   170  }
   171  
   172  // ClientWithNoRedirects makes a new http client which won't follow redirects
   173  func ClientWithNoRedirects(c *http.Client) *http.Client {
   174  	clientCopy := *c
   175  	clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error {
   176  		return http.ErrUseLastResponse
   177  	}
   178  	return &clientCopy
   179  }
   180  
   181  // Call makes the call and returns the http.Response
   182  //
   183  // if err == nil then resp.Body will need to be closed unless
   184  // opt.NoResponse is set
   185  //
   186  // if err != nil then resp.Body will have been closed
   187  //
   188  // it will return resp if at all possible, even if err is set
   189  func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, err error) {
   190  	api.mu.RLock()
   191  	defer api.mu.RUnlock()
   192  	if opts == nil {
   193  		return nil, errors.New("call() called with nil opts")
   194  	}
   195  	url := api.rootURL
   196  	if opts.RootURL != "" {
   197  		url = opts.RootURL
   198  	}
   199  	if url == "" {
   200  		return nil, errors.New("RootURL not set")
   201  	}
   202  	url += opts.Path
   203  	if opts.Parameters != nil && len(opts.Parameters) > 0 {
   204  		url += "?" + opts.Parameters.Encode()
   205  	}
   206  	body := readers.NoCloser(opts.Body)
   207  	// If length is set and zero then nil out the body to stop use
   208  	// use of chunked encoding and insert a "Content-Length: 0"
   209  	// header.
   210  	//
   211  	// If we don't do this we get "Content-Length" headers for all
   212  	// files except 0 length files.
   213  	if opts.ContentLength != nil && *opts.ContentLength == 0 {
   214  		body = nil
   215  	}
   216  	req, err := http.NewRequest(opts.Method, url, body)
   217  	if err != nil {
   218  		return
   219  	}
   220  	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
   221  	headers := make(map[string]string)
   222  	// Set default headers
   223  	for k, v := range api.headers {
   224  		headers[k] = v
   225  	}
   226  	if opts.ContentType != "" {
   227  		headers["Content-Type"] = opts.ContentType
   228  	}
   229  	if opts.ContentLength != nil {
   230  		req.ContentLength = *opts.ContentLength
   231  	}
   232  	if opts.ContentRange != "" {
   233  		headers["Content-Range"] = opts.ContentRange
   234  	}
   235  	if len(opts.TransferEncoding) != 0 {
   236  		req.TransferEncoding = opts.TransferEncoding
   237  	}
   238  	if opts.Close {
   239  		req.Close = true
   240  	}
   241  	// Set any extra headers
   242  	if opts.ExtraHeaders != nil {
   243  		for k, v := range opts.ExtraHeaders {
   244  			headers[k] = v
   245  		}
   246  	}
   247  	// add any options to the headers
   248  	fs.OpenOptionAddHeaders(opts.Options, headers)
   249  	// Now set the headers
   250  	for k, v := range headers {
   251  		if k != "" && v != "" {
   252  			if k[0] == '*' {
   253  				// Add non-canonical version if header starts with *
   254  				k = k[1:]
   255  				req.Header[k] = append(req.Header[k], v)
   256  			} else {
   257  				req.Header.Add(k, v)
   258  			}
   259  		}
   260  	}
   261  
   262  	if opts.UserName != "" || opts.Password != "" {
   263  		req.SetBasicAuth(opts.UserName, opts.Password)
   264  	}
   265  	var c *http.Client
   266  	if opts.NoRedirect {
   267  		c = ClientWithNoRedirects(api.c)
   268  	} else {
   269  		c = api.c
   270  	}
   271  	if api.signer != nil {
   272  		api.mu.RUnlock()
   273  		err = api.signer(req)
   274  		api.mu.RLock()
   275  		if err != nil {
   276  			return nil, errors.Wrap(err, "signer failed")
   277  		}
   278  	}
   279  	api.mu.RUnlock()
   280  	resp, err = c.Do(req)
   281  	api.mu.RLock()
   282  	if err != nil {
   283  		return nil, err
   284  	}
   285  	if !opts.IgnoreStatus {
   286  		if resp.StatusCode < 200 || resp.StatusCode > 299 {
   287  			err = api.errorHandler(resp)
   288  			if err.Error() == "" {
   289  				// replace empty errors with something
   290  				err = errors.Errorf("http error %d: %v", resp.StatusCode, resp.Status)
   291  			}
   292  			return resp, err
   293  		}
   294  	}
   295  	if opts.NoResponse {
   296  		return resp, resp.Body.Close()
   297  	}
   298  	return resp, nil
   299  }
   300  
   301  // MultipartUpload creates an io.Reader which produces an encoded a
   302  // multipart form upload from the params passed in and the  passed in
   303  //
   304  // in - the body of the file (may be nil)
   305  // params - the form parameters
   306  // fileName - is the name of the attached file
   307  // contentName - the name of the parameter for the file
   308  //
   309  // the int64 returned is the overhead in addition to the file contents, in case Content-Length is required
   310  //
   311  // NB This doesn't allow setting the content type of the attachment
   312  func MultipartUpload(in io.Reader, params url.Values, contentName, fileName string) (io.ReadCloser, string, int64, error) {
   313  	bodyReader, bodyWriter := io.Pipe()
   314  	writer := multipart.NewWriter(bodyWriter)
   315  	contentType := writer.FormDataContentType()
   316  
   317  	// Create a Multipart Writer as base for calculating the Content-Length
   318  	buf := &bytes.Buffer{}
   319  	dummyMultipartWriter := multipart.NewWriter(buf)
   320  	err := dummyMultipartWriter.SetBoundary(writer.Boundary())
   321  	if err != nil {
   322  		return nil, "", 0, err
   323  	}
   324  
   325  	for key, vals := range params {
   326  		for _, val := range vals {
   327  			err := dummyMultipartWriter.WriteField(key, val)
   328  			if err != nil {
   329  				return nil, "", 0, err
   330  			}
   331  		}
   332  	}
   333  	if in != nil {
   334  		_, err = dummyMultipartWriter.CreateFormFile(contentName, fileName)
   335  		if err != nil {
   336  			return nil, "", 0, err
   337  		}
   338  	}
   339  
   340  	err = dummyMultipartWriter.Close()
   341  	if err != nil {
   342  		return nil, "", 0, err
   343  	}
   344  
   345  	multipartLength := int64(buf.Len())
   346  
   347  	// Pump the data in the background
   348  	go func() {
   349  		var err error
   350  
   351  		for key, vals := range params {
   352  			for _, val := range vals {
   353  				err = writer.WriteField(key, val)
   354  				if err != nil {
   355  					_ = bodyWriter.CloseWithError(errors.Wrap(err, "create metadata part"))
   356  					return
   357  				}
   358  			}
   359  		}
   360  
   361  		if in != nil {
   362  			part, err := writer.CreateFormFile(contentName, fileName)
   363  			if err != nil {
   364  				_ = bodyWriter.CloseWithError(errors.Wrap(err, "failed to create form file"))
   365  				return
   366  			}
   367  
   368  			_, err = io.Copy(part, in)
   369  			if err != nil {
   370  				_ = bodyWriter.CloseWithError(errors.Wrap(err, "failed to copy data"))
   371  				return
   372  			}
   373  		}
   374  
   375  		err = writer.Close()
   376  		if err != nil {
   377  			_ = bodyWriter.CloseWithError(errors.Wrap(err, "failed to close form"))
   378  			return
   379  		}
   380  
   381  		_ = bodyWriter.Close()
   382  	}()
   383  
   384  	return bodyReader, contentType, multipartLength, nil
   385  }
   386  
   387  // CallJSON runs Call and decodes the body as a JSON object into response (if not nil)
   388  //
   389  // If request is not nil then it will be JSON encoded as the body of the request
   390  //
   391  // If response is not nil then the response will be JSON decoded into
   392  // it and resp.Body will be closed.
   393  //
   394  // If response is nil then the resp.Body will be closed only if
   395  // opts.NoResponse is set.
   396  //
   397  // If (opts.MultipartParams or opts.MultipartContentName) and
   398  // opts.Body are set then CallJSON will do a multipart upload with a
   399  // file attached.  opts.MultipartContentName is the name of the
   400  // parameter and opts.MultipartFileName is the name of the file.  If
   401  // MultpartContentName is set, and request != nil is supplied, then
   402  // the request will be marshalled into JSON and added to the form with
   403  // parameter name MultipartMetadataName.
   404  //
   405  // It will return resp if at all possible, even if err is set
   406  func (api *Client) CallJSON(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
   407  	return api.callCodec(ctx, opts, request, response, json.Marshal, DecodeJSON, "application/json")
   408  }
   409  
   410  // CallXML runs Call and decodes the body as a XML object into response (if not nil)
   411  //
   412  // If request is not nil then it will be XML encoded as the body of the request
   413  //
   414  // If response is not nil then the response will be XML decoded into
   415  // it and resp.Body will be closed.
   416  //
   417  // If response is nil then the resp.Body will be closed only if
   418  // opts.NoResponse is set.
   419  //
   420  // See CallJSON for a description of MultipartParams and related opts
   421  //
   422  // It will return resp if at all possible, even if err is set
   423  func (api *Client) CallXML(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
   424  	return api.callCodec(ctx, opts, request, response, xml.Marshal, DecodeXML, "application/xml")
   425  }
   426  
   427  type marshalFn func(v interface{}) ([]byte, error)
   428  type decodeFn func(resp *http.Response, result interface{}) (err error)
   429  
   430  func (api *Client) callCodec(ctx context.Context, opts *Opts, request interface{}, response interface{}, marshal marshalFn, decode decodeFn, contentType string) (resp *http.Response, err error) {
   431  	var requestBody []byte
   432  	// Marshal the request if given
   433  	if request != nil {
   434  		requestBody, err = marshal(request)
   435  		if err != nil {
   436  			return nil, err
   437  		}
   438  		// Set the body up as a marshalled object if no body passed in
   439  		if opts.Body == nil {
   440  			opts = opts.Copy()
   441  			opts.ContentType = contentType
   442  			opts.Body = bytes.NewBuffer(requestBody)
   443  		}
   444  	}
   445  	if opts.MultipartParams != nil || opts.MultipartContentName != "" {
   446  		params := opts.MultipartParams
   447  		if params == nil {
   448  			params = url.Values{}
   449  		}
   450  		if opts.MultipartMetadataName != "" {
   451  			params.Add(opts.MultipartMetadataName, string(requestBody))
   452  		}
   453  		opts = opts.Copy()
   454  
   455  		var overhead int64
   456  		opts.Body, opts.ContentType, overhead, err = MultipartUpload(opts.Body, params, opts.MultipartContentName, opts.MultipartFileName)
   457  		if err != nil {
   458  			return nil, err
   459  		}
   460  		if opts.ContentLength != nil {
   461  			*opts.ContentLength += overhead
   462  		}
   463  	}
   464  	resp, err = api.Call(ctx, opts)
   465  	if err != nil {
   466  		return resp, err
   467  	}
   468  	// if opts.NoResponse is set, resp.Body will have been closed by Call()
   469  	if response == nil || opts.NoResponse {
   470  		return resp, nil
   471  	}
   472  	err = decode(resp, response)
   473  	return resp, err
   474  }