github.com/jxskiss/gopkg@v0.17.3/easy/http.go (about)

     1  package easy
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/xml"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"log"
    11  	"net"
    12  	"net/http"
    13  	"net/http/httputil"
    14  	"net/url"
    15  	"regexp"
    16  	"strings"
    17  	"time"
    18  
    19  	"github.com/jxskiss/gopkg/internal/unsafeheader"
    20  	"github.com/jxskiss/gopkg/json"
    21  )
    22  
    23  var (
    24  	hdrContentTypeKey = http.CanonicalHeaderKey("Content-Type")
    25  	contentTypeJSON   = "application/json"
    26  	contentTypeXML    = "application/xml"
    27  	contentTypeForm   = "application/x-www-form-urlencoded"
    28  
    29  	jsonCheck = regexp.MustCompile(`(?i:(application|text)/(json|.*\+json|json\-.*)(;|$))`)
    30  	xmlCheck  = regexp.MustCompile(`(?i:(application|text)/(xml|.*\+xml)(;|$))`)
    31  )
    32  
    33  // SingleJoin joins the given text segments using sep.
    34  // No matter whether a segment begins or ends with sep or not, it
    35  // guarantees that only one sep appears between two segments.
    36  func SingleJoin(sep string, text ...string) string {
    37  	if len(text) == 0 {
    38  		return ""
    39  	}
    40  	result := text[0]
    41  	for _, next := range text[1:] {
    42  		asep := strings.HasSuffix(result, sep)
    43  		bsep := strings.HasPrefix(next, sep)
    44  		switch {
    45  		case asep && bsep:
    46  			result += next[len(sep):]
    47  		case !asep && !bsep:
    48  			result += sep + next
    49  		default:
    50  			result += next
    51  		}
    52  	}
    53  	return result
    54  }
    55  
    56  // SlashJoin joins the given path segments using "/".
    57  // No matter whether a segment begins or ends with "/" or not, it guarantees
    58  // that only one "/" appears between two segments.
    59  func SlashJoin(path ...string) string {
    60  	return SingleJoin("/", path...)
    61  }
    62  
    63  // JSONToReader converts an json encodeable object to an io.Reader.
    64  // If obj cannot be marshaled as JSON, it reports the error.
    65  func JSONToReader(obj interface{}) (io.Reader, error) {
    66  	b, err := json.Marshal(obj)
    67  	if err != nil {
    68  		return nil, err
    69  	}
    70  	return bytes.NewBuffer(b), nil
    71  }
    72  
    73  // DecodeJSON decodes a json value from r.
    74  func DecodeJSON(r io.Reader, v interface{}) error {
    75  	data, err := ioutil.ReadAll(r)
    76  	if err != nil {
    77  		return err
    78  	}
    79  	return json.Unmarshal(data, v)
    80  }
    81  
    82  // IsJSONType method is to check JSON content type or not.
    83  func IsJSONType(contentType string) bool {
    84  	return jsonCheck.MatchString(contentType)
    85  }
    86  
    87  // XMLToReader converts an XML encodeable object to an io.Reader.
    88  // If obj cannot be marshaled as XML, it reports the error.
    89  func XMLToReader(obj interface{}) (io.Reader, error) {
    90  	b, err := xml.Marshal(obj)
    91  	if err != nil {
    92  		return nil, err
    93  	}
    94  	return bytes.NewBuffer(b), nil
    95  }
    96  
    97  // DecodeXML decodes a XML value from r.
    98  func DecodeXML(r io.Reader, v interface{}) error {
    99  	data, err := ioutil.ReadAll(r)
   100  	if err != nil {
   101  		return err
   102  	}
   103  	return xml.Unmarshal(data, v)
   104  }
   105  
   106  // IsXMLType method is to check XML content type or not.
   107  func IsXMLType(contentType string) bool {
   108  	return xmlCheck.MatchString(contentType)
   109  }
   110  
   111  // Request represents a request and options to send with the DoRequest function.
   112  type Request struct {
   113  
   114  	// Req should be a fully prepared http Request to sent, if not nil,
   115  	// the following `Method`, `URL`, `Params`, `JSON`, `XML`, `Form`, `Body`
   116  	// will be ignored.
   117  	//
   118  	// If Req is nil, it will be filled using the following data `Method`,
   119  	// `URL`, `Params`, `JSON`, `XML`, `Form`, `Body` to construct the `http.Request`.
   120  	//
   121  	// When building http body, the priority is JSON > XML > Form > Body.
   122  	Req *http.Request
   123  
   124  	// Method specifies the verb for the http request, it's optional,
   125  	// default is "GET".
   126  	Method string
   127  
   128  	// URL specifies the url to make the http request.
   129  	// It's required if Req is nil.
   130  	URL string
   131  
   132  	// Params specifies optional params to merge with URL, it must be one of
   133  	// the following types:
   134  	// - map[string]string
   135  	// - map[string][]string
   136  	// - map[string]interface{}
   137  	// - url.Values
   138  	Params interface{}
   139  
   140  	// JSON specifies optional body data for request which can take body,
   141  	// the content-type will be "application/json", it must be one of
   142  	// the following types:
   143  	// - io.Reader
   144  	// - []byte (will be wrapped with bytes.NewReader)
   145  	// - string (will be wrapped with strings.NewReader)
   146  	// - interface{} (will be marshaled with json.Marshal)
   147  	JSON interface{}
   148  
   149  	// XML specifies optional body data for request which can take body,
   150  	// the content-type will be "application/xml", it must be one of
   151  	// the following types:
   152  	// - io.Reader
   153  	// - []byte (will be wrapped with bytes.NewReader)
   154  	// - string (will be wrapped with strings.NewReader)
   155  	// - interface{} (will be marshaled with xml.Marshal)
   156  	XML interface{}
   157  
   158  	// Form specifies optional body data for request which can take body,
   159  	// the content-type will be "application/x-www-form-urlencoded",
   160  	// it must be one of the following types:
   161  	// - io.Reader
   162  	// - []byte (will be wrapped with bytes.NewReader)
   163  	// - string (will be wrapped with strings.NewReader)
   164  	// - url.Values (will be encoded and wrapped as io.Reader)
   165  	// - map[string]string (will be converted to url.Values)
   166  	// - map[string][]string (will be converted to url.Values)
   167  	// - map[string]interface{} (will be converted to url.Values)
   168  	Form interface{}
   169  
   170  	// Body specifies optional body data for request which can take body,
   171  	// the content-type will be detected from the content (may be incorrect),
   172  	// it should be one of the following types:
   173  	// - io.Reader
   174  	// - []byte (will be wrapped with bytes.NewReader)
   175  	// - string (will be wrapped with strings.NewReader)
   176  	Body interface{}
   177  
   178  	// Headers will be copied to the request before sent.
   179  	//
   180  	// If "Content-Type" presents, it will replace the default value
   181  	// set by `JSON`, `XML`, `Form`, or `Body`.
   182  	Headers map[string]string
   183  
   184  	// Resp specifies an optional destination to unmarshal the response data.
   185  	// if `Unmarshal` is not provided, the header "Content-Type" will be used to
   186  	// detect XML content, else `json.Unmarshal` will be used.
   187  	Resp interface{}
   188  
   189  	// Unmarshal specifies an optional function to unmarshal the response data.
   190  	Unmarshal func([]byte, interface{}) error
   191  
   192  	// Context specifies an optional context.Context to use with http request.
   193  	Context context.Context
   194  
   195  	// Timeout specifies an optional timeout of the http request, if
   196  	// timeout > 0, the request will be attached an timeout context.Context.
   197  	// `Timeout` takes higher priority than `Context`, it both available, only
   198  	// `Timeout` takes effect.
   199  	Timeout time.Duration
   200  
   201  	// Client specifies an optional http.Client to do the request, instead of
   202  	// the default http client.
   203  	Client *http.Client
   204  
   205  	// DisableRedirect tells the http client don't follow response redirection.
   206  	DisableRedirect bool
   207  
   208  	// DumpRequest makes the http request being logged before sent.
   209  	DumpRequest bool
   210  
   211  	// DumpResponse makes the http response being logged after received.
   212  	DumpResponse bool
   213  
   214  	// RaiseForStatus tells `DoRequest` to report an error if the response
   215  	// status code >= 400. The error will be formatted as "unexpected status: <STATUS>".
   216  	RaiseForStatus bool
   217  }
   218  
   219  func (p *Request) buildClient() *http.Client {
   220  	if p.Client == nil && !p.DisableRedirect {
   221  		return NoHttp2Client
   222  	}
   223  	var client = *NoHttp2Client
   224  	if p.Client != nil {
   225  		client = *p.Client
   226  	}
   227  	if p.DisableRedirect {
   228  		client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
   229  			return http.ErrUseLastResponse
   230  		}
   231  	}
   232  	return &client
   233  }
   234  
   235  func (p *Request) prepareRequest(method string) (err error) {
   236  	if p.Req != nil {
   237  		return nil
   238  	}
   239  	reqURL := p.URL
   240  	if p.Params != nil {
   241  		reqURL, err = mergeQuery(reqURL, p.Params)
   242  		if err != nil {
   243  			return err
   244  		}
   245  	}
   246  	if method == "" {
   247  		method = p.Method
   248  	}
   249  	if method == "" || method == "GET" {
   250  		p.Req, err = http.NewRequest(method, reqURL, nil)
   251  		return
   252  	}
   253  
   254  	var body io.Reader
   255  	var contentType string
   256  
   257  	if p.JSON != nil { // JSON
   258  		body, err = p.makeBody(p.JSON, json.Marshal)
   259  		contentType = contentTypeJSON
   260  	} else if p.XML != nil { // XML
   261  		body, err = p.makeBody(p.XML, xml.Marshal)
   262  		contentType = contentTypeXML
   263  	} else if p.Form != nil { // urlencoded form
   264  		body, err = p.makeBody(p.Form, marshalForm)
   265  		contentType = contentTypeForm
   266  	} else if p.Body != nil { // detect content-type from the body data
   267  		var bodyBuf []byte
   268  		switch data := p.Body.(type) {
   269  		case io.Reader:
   270  			bodyBuf, err = ioutil.ReadAll(data)
   271  			if err != nil {
   272  				return err
   273  			}
   274  		case []byte:
   275  			bodyBuf = data
   276  		case string:
   277  			bodyBuf = unsafeheader.StringToBytes(data)
   278  		default:
   279  			err = fmt.Errorf("unsupported body data type: %T", data)
   280  			return err
   281  		}
   282  		body = bytes.NewReader(bodyBuf)
   283  		if p.Headers[hdrContentTypeKey] == "" {
   284  			contentType = http.DetectContentType(bodyBuf)
   285  		}
   286  	} // else no body data
   287  
   288  	if err != nil {
   289  		return err
   290  	}
   291  	p.Req, err = http.NewRequest(method, p.URL, body)
   292  	if err != nil {
   293  		return err
   294  	}
   295  	if contentType != "" {
   296  		p.Req.Header.Set(hdrContentTypeKey, contentType)
   297  	}
   298  	return
   299  }
   300  
   301  func mergeQuery(reqURL string, params interface{}) (string, error) {
   302  	parsed, err := url.Parse(reqURL)
   303  	if err != nil {
   304  		return "", err
   305  	}
   306  	query, err := url.ParseQuery(parsed.RawQuery)
   307  	if err != nil {
   308  		return "", err
   309  	}
   310  	switch params := params.(type) {
   311  	case map[string]string:
   312  		for k, v := range params {
   313  			query.Add(k, v)
   314  		}
   315  	case map[string][]string:
   316  		for k, values := range params {
   317  			for _, v := range values {
   318  				query.Add(k, v)
   319  			}
   320  		}
   321  	case map[string]interface{}:
   322  		for k, v := range params {
   323  			switch value := v.(type) {
   324  			case string:
   325  				query.Add(k, fmt.Sprintf(value))
   326  			case []string:
   327  				for _, v := range value {
   328  					query.Add(k, v)
   329  				}
   330  			default:
   331  				query.Add(k, fmt.Sprint(v))
   332  			}
   333  		}
   334  	case url.Values:
   335  		for k, values := range params {
   336  			for _, v := range values {
   337  				query.Add(k, v)
   338  			}
   339  		}
   340  	default:
   341  		err = fmt.Errorf("unsupported params type: %T", params)
   342  		return "", err
   343  	}
   344  	parsed.RawQuery = query.Encode()
   345  	return parsed.String(), nil
   346  }
   347  
   348  func marshalForm(v interface{}) ([]byte, error) {
   349  	var form url.Values
   350  	switch data := v.(type) {
   351  	case url.Values:
   352  		form = data
   353  	case map[string][]string:
   354  		form = data
   355  	case map[string]string:
   356  		form = make(url.Values, len(data))
   357  		for k, v := range data {
   358  			form[k] = []string{v}
   359  		}
   360  	case map[string]interface{}:
   361  		form = make(url.Values, len(data))
   362  		for k, v := range data {
   363  			switch value := v.(type) {
   364  			case string:
   365  				form[k] = []string{value}
   366  			case []string:
   367  				form[k] = value
   368  			default:
   369  				form[k] = []string{fmt.Sprint(v)}
   370  			}
   371  		}
   372  	}
   373  	if form == nil {
   374  		err := fmt.Errorf("unsupported form data type: %T", v)
   375  		return nil, err
   376  	}
   377  	encoded := form.Encode()
   378  	buf := unsafeheader.StringToBytes(encoded)
   379  	return buf, nil
   380  }
   381  
   382  type marshalFunc func(interface{}) ([]byte, error)
   383  
   384  func (p *Request) makeBody(data interface{}, marshal marshalFunc) (io.Reader, error) {
   385  	var body io.Reader
   386  	switch x := data.(type) {
   387  	case io.Reader:
   388  		body = x
   389  	case []byte:
   390  		body = bytes.NewReader(x)
   391  	case string:
   392  		body = strings.NewReader(x)
   393  	default:
   394  		buf, err := marshal(data)
   395  		if err != nil {
   396  			return nil, err
   397  		}
   398  		body = bytes.NewReader(buf)
   399  	}
   400  	return body, nil
   401  }
   402  
   403  func (p *Request) prepareHeaders() {
   404  	if p.Req == nil {
   405  		return
   406  	}
   407  	for k, v := range p.Headers {
   408  		p.Req.Header.Set(k, v)
   409  	}
   410  }
   411  
   412  // DoRequest is a convenient function to send request and control redirect
   413  // and debug options. If `Request.Resp` is provided, it will be used as
   414  // destination to try to unmarshal the response body.
   415  //
   416  // Trade-off was taken to balance simplicity and convenience of the function.
   417  //
   418  // For more powerful controls of a http request and handy utilities,
   419  // you may take a look at the awesome library `https://github.com/go-resty/resty/`.
   420  func DoRequest(req *Request) (respContent []byte, status int, err error) {
   421  	if err = req.prepareRequest(""); err != nil {
   422  		return
   423  	}
   424  	req.prepareHeaders()
   425  
   426  	httpReq := req.Req
   427  	if req.Context != nil {
   428  		httpReq = httpReq.WithContext(req.Context)
   429  	}
   430  	if req.Timeout > 0 {
   431  		timeoutCtx, cancel := context.WithTimeout(httpReq.Context(), req.Timeout)
   432  		defer cancel()
   433  		httpReq = httpReq.WithContext(timeoutCtx)
   434  	}
   435  	if req.DumpRequest {
   436  		var dump []byte
   437  		dump, err = httputil.DumpRequestOut(httpReq, true)
   438  		if err != nil {
   439  			return
   440  		}
   441  		log.Printf("dump http request:\n%s", dump)
   442  	}
   443  
   444  	httpClient := req.buildClient()
   445  	httpResp, err := httpClient.Do(httpReq)
   446  	if err != nil {
   447  		return
   448  	}
   449  	defer httpResp.Body.Close()
   450  
   451  	status = httpResp.StatusCode
   452  	if req.DumpResponse {
   453  		var dump []byte
   454  		dump, err = httputil.DumpResponse(httpResp, true)
   455  		if err != nil {
   456  			return
   457  		}
   458  		log.Printf("dump http response:\n%s", dump)
   459  	}
   460  
   461  	respContent, err = ioutil.ReadAll(httpResp.Body)
   462  	if err != nil {
   463  		return
   464  	}
   465  	if req.RaiseForStatus {
   466  		if httpResp.StatusCode >= 400 {
   467  			err = fmt.Errorf("unexpected status: %v", httpResp.Status)
   468  			return
   469  		}
   470  	}
   471  
   472  	if req.Resp != nil && len(respContent) > 0 {
   473  		unmarshal := req.Unmarshal
   474  		if unmarshal == nil {
   475  			ct := httpResp.Header.Get(hdrContentTypeKey)
   476  			if IsXMLType(ct) {
   477  				unmarshal = xml.Unmarshal
   478  			}
   479  			// default: JSON
   480  			if unmarshal == nil {
   481  				unmarshal = json.Unmarshal
   482  			}
   483  		}
   484  		err = unmarshal(respContent, req.Resp)
   485  		if err != nil {
   486  			return
   487  		}
   488  	}
   489  	return
   490  }
   491  
   492  var zeroDialer net.Dialer
   493  
   494  // NoHttp2Client disables HTTP/2 feature by specifying a custom DialContext
   495  // function, otherwise it behaves exactly same with http.NoHttp2Client.
   496  //
   497  // HTTP/2 support in golang has many problematic corner cases where dead
   498  // connections would be kept and used in connection pools. See:
   499  //
   500  //   https://github.com/golang/go/issues/32388
   501  //   https://github.com/golang/go/issues/39337
   502  //   https://github.com/golang/go/issues/39750
   503  //
   504  // The deadlock reported by issue 32388 is observed in our production deployment,
   505  // thus we disable HTTP/2 to help to get a better sleep.
   506  var NoHttp2Client = &http.Client{
   507  	Transport: &http.Transport{
   508  		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
   509  			return zeroDialer.DialContext(ctx, network, addr)
   510  		},
   511  		Dial: func(network, addr string) (net.Conn, error) {
   512  			return zeroDialer.Dial(network, addr)
   513  		},
   514  	},
   515  }