github.com/jxskiss/gopkg@v0.17.3/easy/http.go (about) 1 package easy 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/xml" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "log" 11 "net" 12 "net/http" 13 "net/http/httputil" 14 "net/url" 15 "regexp" 16 "strings" 17 "time" 18 19 "github.com/jxskiss/gopkg/internal/unsafeheader" 20 "github.com/jxskiss/gopkg/json" 21 ) 22 23 var ( 24 hdrContentTypeKey = http.CanonicalHeaderKey("Content-Type") 25 contentTypeJSON = "application/json" 26 contentTypeXML = "application/xml" 27 contentTypeForm = "application/x-www-form-urlencoded" 28 29 jsonCheck = regexp.MustCompile(`(?i:(application|text)/(json|.*\+json|json\-.*)(;|$))`) 30 xmlCheck = regexp.MustCompile(`(?i:(application|text)/(xml|.*\+xml)(;|$))`) 31 ) 32 33 // SingleJoin joins the given text segments using sep. 34 // No matter whether a segment begins or ends with sep or not, it 35 // guarantees that only one sep appears between two segments. 36 func SingleJoin(sep string, text ...string) string { 37 if len(text) == 0 { 38 return "" 39 } 40 result := text[0] 41 for _, next := range text[1:] { 42 asep := strings.HasSuffix(result, sep) 43 bsep := strings.HasPrefix(next, sep) 44 switch { 45 case asep && bsep: 46 result += next[len(sep):] 47 case !asep && !bsep: 48 result += sep + next 49 default: 50 result += next 51 } 52 } 53 return result 54 } 55 56 // SlashJoin joins the given path segments using "/". 57 // No matter whether a segment begins or ends with "/" or not, it guarantees 58 // that only one "/" appears between two segments. 59 func SlashJoin(path ...string) string { 60 return SingleJoin("/", path...) 61 } 62 63 // JSONToReader converts an json encodeable object to an io.Reader. 64 // If obj cannot be marshaled as JSON, it reports the error. 65 func JSONToReader(obj interface{}) (io.Reader, error) { 66 b, err := json.Marshal(obj) 67 if err != nil { 68 return nil, err 69 } 70 return bytes.NewBuffer(b), nil 71 } 72 73 // DecodeJSON decodes a json value from r. 74 func DecodeJSON(r io.Reader, v interface{}) error { 75 data, err := ioutil.ReadAll(r) 76 if err != nil { 77 return err 78 } 79 return json.Unmarshal(data, v) 80 } 81 82 // IsJSONType method is to check JSON content type or not. 83 func IsJSONType(contentType string) bool { 84 return jsonCheck.MatchString(contentType) 85 } 86 87 // XMLToReader converts an XML encodeable object to an io.Reader. 88 // If obj cannot be marshaled as XML, it reports the error. 89 func XMLToReader(obj interface{}) (io.Reader, error) { 90 b, err := xml.Marshal(obj) 91 if err != nil { 92 return nil, err 93 } 94 return bytes.NewBuffer(b), nil 95 } 96 97 // DecodeXML decodes a XML value from r. 98 func DecodeXML(r io.Reader, v interface{}) error { 99 data, err := ioutil.ReadAll(r) 100 if err != nil { 101 return err 102 } 103 return xml.Unmarshal(data, v) 104 } 105 106 // IsXMLType method is to check XML content type or not. 107 func IsXMLType(contentType string) bool { 108 return xmlCheck.MatchString(contentType) 109 } 110 111 // Request represents a request and options to send with the DoRequest function. 112 type Request struct { 113 114 // Req should be a fully prepared http Request to sent, if not nil, 115 // the following `Method`, `URL`, `Params`, `JSON`, `XML`, `Form`, `Body` 116 // will be ignored. 117 // 118 // If Req is nil, it will be filled using the following data `Method`, 119 // `URL`, `Params`, `JSON`, `XML`, `Form`, `Body` to construct the `http.Request`. 120 // 121 // When building http body, the priority is JSON > XML > Form > Body. 122 Req *http.Request 123 124 // Method specifies the verb for the http request, it's optional, 125 // default is "GET". 126 Method string 127 128 // URL specifies the url to make the http request. 129 // It's required if Req is nil. 130 URL string 131 132 // Params specifies optional params to merge with URL, it must be one of 133 // the following types: 134 // - map[string]string 135 // - map[string][]string 136 // - map[string]interface{} 137 // - url.Values 138 Params interface{} 139 140 // JSON specifies optional body data for request which can take body, 141 // the content-type will be "application/json", it must be one of 142 // the following types: 143 // - io.Reader 144 // - []byte (will be wrapped with bytes.NewReader) 145 // - string (will be wrapped with strings.NewReader) 146 // - interface{} (will be marshaled with json.Marshal) 147 JSON interface{} 148 149 // XML specifies optional body data for request which can take body, 150 // the content-type will be "application/xml", it must be one of 151 // the following types: 152 // - io.Reader 153 // - []byte (will be wrapped with bytes.NewReader) 154 // - string (will be wrapped with strings.NewReader) 155 // - interface{} (will be marshaled with xml.Marshal) 156 XML interface{} 157 158 // Form specifies optional body data for request which can take body, 159 // the content-type will be "application/x-www-form-urlencoded", 160 // it must be one of the following types: 161 // - io.Reader 162 // - []byte (will be wrapped with bytes.NewReader) 163 // - string (will be wrapped with strings.NewReader) 164 // - url.Values (will be encoded and wrapped as io.Reader) 165 // - map[string]string (will be converted to url.Values) 166 // - map[string][]string (will be converted to url.Values) 167 // - map[string]interface{} (will be converted to url.Values) 168 Form interface{} 169 170 // Body specifies optional body data for request which can take body, 171 // the content-type will be detected from the content (may be incorrect), 172 // it should be one of the following types: 173 // - io.Reader 174 // - []byte (will be wrapped with bytes.NewReader) 175 // - string (will be wrapped with strings.NewReader) 176 Body interface{} 177 178 // Headers will be copied to the request before sent. 179 // 180 // If "Content-Type" presents, it will replace the default value 181 // set by `JSON`, `XML`, `Form`, or `Body`. 182 Headers map[string]string 183 184 // Resp specifies an optional destination to unmarshal the response data. 185 // if `Unmarshal` is not provided, the header "Content-Type" will be used to 186 // detect XML content, else `json.Unmarshal` will be used. 187 Resp interface{} 188 189 // Unmarshal specifies an optional function to unmarshal the response data. 190 Unmarshal func([]byte, interface{}) error 191 192 // Context specifies an optional context.Context to use with http request. 193 Context context.Context 194 195 // Timeout specifies an optional timeout of the http request, if 196 // timeout > 0, the request will be attached an timeout context.Context. 197 // `Timeout` takes higher priority than `Context`, it both available, only 198 // `Timeout` takes effect. 199 Timeout time.Duration 200 201 // Client specifies an optional http.Client to do the request, instead of 202 // the default http client. 203 Client *http.Client 204 205 // DisableRedirect tells the http client don't follow response redirection. 206 DisableRedirect bool 207 208 // DumpRequest makes the http request being logged before sent. 209 DumpRequest bool 210 211 // DumpResponse makes the http response being logged after received. 212 DumpResponse bool 213 214 // RaiseForStatus tells `DoRequest` to report an error if the response 215 // status code >= 400. The error will be formatted as "unexpected status: <STATUS>". 216 RaiseForStatus bool 217 } 218 219 func (p *Request) buildClient() *http.Client { 220 if p.Client == nil && !p.DisableRedirect { 221 return NoHttp2Client 222 } 223 var client = *NoHttp2Client 224 if p.Client != nil { 225 client = *p.Client 226 } 227 if p.DisableRedirect { 228 client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { 229 return http.ErrUseLastResponse 230 } 231 } 232 return &client 233 } 234 235 func (p *Request) prepareRequest(method string) (err error) { 236 if p.Req != nil { 237 return nil 238 } 239 reqURL := p.URL 240 if p.Params != nil { 241 reqURL, err = mergeQuery(reqURL, p.Params) 242 if err != nil { 243 return err 244 } 245 } 246 if method == "" { 247 method = p.Method 248 } 249 if method == "" || method == "GET" { 250 p.Req, err = http.NewRequest(method, reqURL, nil) 251 return 252 } 253 254 var body io.Reader 255 var contentType string 256 257 if p.JSON != nil { // JSON 258 body, err = p.makeBody(p.JSON, json.Marshal) 259 contentType = contentTypeJSON 260 } else if p.XML != nil { // XML 261 body, err = p.makeBody(p.XML, xml.Marshal) 262 contentType = contentTypeXML 263 } else if p.Form != nil { // urlencoded form 264 body, err = p.makeBody(p.Form, marshalForm) 265 contentType = contentTypeForm 266 } else if p.Body != nil { // detect content-type from the body data 267 var bodyBuf []byte 268 switch data := p.Body.(type) { 269 case io.Reader: 270 bodyBuf, err = ioutil.ReadAll(data) 271 if err != nil { 272 return err 273 } 274 case []byte: 275 bodyBuf = data 276 case string: 277 bodyBuf = unsafeheader.StringToBytes(data) 278 default: 279 err = fmt.Errorf("unsupported body data type: %T", data) 280 return err 281 } 282 body = bytes.NewReader(bodyBuf) 283 if p.Headers[hdrContentTypeKey] == "" { 284 contentType = http.DetectContentType(bodyBuf) 285 } 286 } // else no body data 287 288 if err != nil { 289 return err 290 } 291 p.Req, err = http.NewRequest(method, p.URL, body) 292 if err != nil { 293 return err 294 } 295 if contentType != "" { 296 p.Req.Header.Set(hdrContentTypeKey, contentType) 297 } 298 return 299 } 300 301 func mergeQuery(reqURL string, params interface{}) (string, error) { 302 parsed, err := url.Parse(reqURL) 303 if err != nil { 304 return "", err 305 } 306 query, err := url.ParseQuery(parsed.RawQuery) 307 if err != nil { 308 return "", err 309 } 310 switch params := params.(type) { 311 case map[string]string: 312 for k, v := range params { 313 query.Add(k, v) 314 } 315 case map[string][]string: 316 for k, values := range params { 317 for _, v := range values { 318 query.Add(k, v) 319 } 320 } 321 case map[string]interface{}: 322 for k, v := range params { 323 switch value := v.(type) { 324 case string: 325 query.Add(k, fmt.Sprintf(value)) 326 case []string: 327 for _, v := range value { 328 query.Add(k, v) 329 } 330 default: 331 query.Add(k, fmt.Sprint(v)) 332 } 333 } 334 case url.Values: 335 for k, values := range params { 336 for _, v := range values { 337 query.Add(k, v) 338 } 339 } 340 default: 341 err = fmt.Errorf("unsupported params type: %T", params) 342 return "", err 343 } 344 parsed.RawQuery = query.Encode() 345 return parsed.String(), nil 346 } 347 348 func marshalForm(v interface{}) ([]byte, error) { 349 var form url.Values 350 switch data := v.(type) { 351 case url.Values: 352 form = data 353 case map[string][]string: 354 form = data 355 case map[string]string: 356 form = make(url.Values, len(data)) 357 for k, v := range data { 358 form[k] = []string{v} 359 } 360 case map[string]interface{}: 361 form = make(url.Values, len(data)) 362 for k, v := range data { 363 switch value := v.(type) { 364 case string: 365 form[k] = []string{value} 366 case []string: 367 form[k] = value 368 default: 369 form[k] = []string{fmt.Sprint(v)} 370 } 371 } 372 } 373 if form == nil { 374 err := fmt.Errorf("unsupported form data type: %T", v) 375 return nil, err 376 } 377 encoded := form.Encode() 378 buf := unsafeheader.StringToBytes(encoded) 379 return buf, nil 380 } 381 382 type marshalFunc func(interface{}) ([]byte, error) 383 384 func (p *Request) makeBody(data interface{}, marshal marshalFunc) (io.Reader, error) { 385 var body io.Reader 386 switch x := data.(type) { 387 case io.Reader: 388 body = x 389 case []byte: 390 body = bytes.NewReader(x) 391 case string: 392 body = strings.NewReader(x) 393 default: 394 buf, err := marshal(data) 395 if err != nil { 396 return nil, err 397 } 398 body = bytes.NewReader(buf) 399 } 400 return body, nil 401 } 402 403 func (p *Request) prepareHeaders() { 404 if p.Req == nil { 405 return 406 } 407 for k, v := range p.Headers { 408 p.Req.Header.Set(k, v) 409 } 410 } 411 412 // DoRequest is a convenient function to send request and control redirect 413 // and debug options. If `Request.Resp` is provided, it will be used as 414 // destination to try to unmarshal the response body. 415 // 416 // Trade-off was taken to balance simplicity and convenience of the function. 417 // 418 // For more powerful controls of a http request and handy utilities, 419 // you may take a look at the awesome library `https://github.com/go-resty/resty/`. 420 func DoRequest(req *Request) (respContent []byte, status int, err error) { 421 if err = req.prepareRequest(""); err != nil { 422 return 423 } 424 req.prepareHeaders() 425 426 httpReq := req.Req 427 if req.Context != nil { 428 httpReq = httpReq.WithContext(req.Context) 429 } 430 if req.Timeout > 0 { 431 timeoutCtx, cancel := context.WithTimeout(httpReq.Context(), req.Timeout) 432 defer cancel() 433 httpReq = httpReq.WithContext(timeoutCtx) 434 } 435 if req.DumpRequest { 436 var dump []byte 437 dump, err = httputil.DumpRequestOut(httpReq, true) 438 if err != nil { 439 return 440 } 441 log.Printf("dump http request:\n%s", dump) 442 } 443 444 httpClient := req.buildClient() 445 httpResp, err := httpClient.Do(httpReq) 446 if err != nil { 447 return 448 } 449 defer httpResp.Body.Close() 450 451 status = httpResp.StatusCode 452 if req.DumpResponse { 453 var dump []byte 454 dump, err = httputil.DumpResponse(httpResp, true) 455 if err != nil { 456 return 457 } 458 log.Printf("dump http response:\n%s", dump) 459 } 460 461 respContent, err = ioutil.ReadAll(httpResp.Body) 462 if err != nil { 463 return 464 } 465 if req.RaiseForStatus { 466 if httpResp.StatusCode >= 400 { 467 err = fmt.Errorf("unexpected status: %v", httpResp.Status) 468 return 469 } 470 } 471 472 if req.Resp != nil && len(respContent) > 0 { 473 unmarshal := req.Unmarshal 474 if unmarshal == nil { 475 ct := httpResp.Header.Get(hdrContentTypeKey) 476 if IsXMLType(ct) { 477 unmarshal = xml.Unmarshal 478 } 479 // default: JSON 480 if unmarshal == nil { 481 unmarshal = json.Unmarshal 482 } 483 } 484 err = unmarshal(respContent, req.Resp) 485 if err != nil { 486 return 487 } 488 } 489 return 490 } 491 492 var zeroDialer net.Dialer 493 494 // NoHttp2Client disables HTTP/2 feature by specifying a custom DialContext 495 // function, otherwise it behaves exactly same with http.NoHttp2Client. 496 // 497 // HTTP/2 support in golang has many problematic corner cases where dead 498 // connections would be kept and used in connection pools. See: 499 // 500 // https://github.com/golang/go/issues/32388 501 // https://github.com/golang/go/issues/39337 502 // https://github.com/golang/go/issues/39750 503 // 504 // The deadlock reported by issue 32388 is observed in our production deployment, 505 // thus we disable HTTP/2 to help to get a better sleep. 506 var NoHttp2Client = &http.Client{ 507 Transport: &http.Transport{ 508 DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { 509 return zeroDialer.DialContext(ctx, network, addr) 510 }, 511 Dial: func(network, addr string) (net.Conn, error) { 512 return zeroDialer.Dial(network, addr) 513 }, 514 }, 515 }