github.com/Kolosok86/http@v0.1.2/client.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client. See RFC 7230 through 7235. 6 // 7 // This is the high-level Client interface. 8 // The low-level implementation is in transport.go. 9 10 package http 11 12 import ( 13 "context" 14 "encoding/base64" 15 "errors" 16 "fmt" 17 "io" 18 "log" 19 "net/url" 20 "reflect" 21 "sort" 22 "strings" 23 "sync" 24 "sync/atomic" 25 "time" 26 27 "github.com/Kolosok86/http/internal/ascii" 28 tls "github.com/refraction-networking/utls" 29 ) 30 31 // A Client is an HTTP client. Its zero value (DefaultClient) is a 32 // usable client that uses DefaultTransport. 33 // 34 // The Client's Transport typically has internal state (cached TCP 35 // connections), so Clients should be reused instead of created as 36 // needed. Clients are safe for concurrent use by multiple goroutines. 37 // 38 // A Client is higher-level than a RoundTripper (such as Transport) 39 // and additionally handles HTTP details such as cookies and 40 // redirects. 41 // 42 // When following redirects, the Client will forward all headers set on the 43 // initial Request except: 44 // 45 // • when forwarding sensitive headers like "Authorization", 46 // "WWW-Authenticate", and "Cookie" to untrusted targets. 47 // These headers will be ignored when following a redirect to a domain 48 // that is not a subdomain match or exact match of the initial domain. 49 // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" 50 // will forward the sensitive headers, but a redirect to "bar.com" will not. 51 // 52 // • when forwarding the "Cookie" header with a non-nil cookie Jar. 53 // Since each redirect may mutate the state of the cookie jar, 54 // a redirect may possibly alter a cookie set in the initial request. 55 // When forwarding the "Cookie" header, any mutated cookies will be omitted, 56 // with the expectation that the Jar will insert those mutated cookies 57 // with the updated Values (assuming the origin matches). 58 // If Jar is nil, the initial cookies are forwarded without change. 59 type Client struct { 60 // Transport specifies the mechanism by which individual 61 // HTTP requests are made. 62 // If nil, DefaultTransport is used. 63 Transport RoundTripper 64 65 // CheckRedirect specifies the policy for handling redirects. 66 // If CheckRedirect is not nil, the client calls it before 67 // following an HTTP redirect. The arguments req and via are 68 // the upcoming request and the requests made already, oldest 69 // first. If CheckRedirect returns an error, the Client's Get 70 // method returns both the previous Response (with its Body 71 // closed) and CheckRedirect's error (wrapped in a url.Error) 72 // instead of issuing the Request req. 73 // As a special case, if CheckRedirect returns ErrUseLastResponse, 74 // then the most recent response is returned with its body 75 // unclosed, along with a nil error. 76 // 77 // If CheckRedirect is nil, the Client uses its default policy, 78 // which is to stop after 10 consecutive requests. 79 CheckRedirect func(req *Request, via []*Request) error 80 81 // Jar specifies the cookie jar. 82 // 83 // The Jar is used to insert relevant cookies into every 84 // outbound Request and is updated with the cookie Values 85 // of every inbound Response. The Jar is consulted for every 86 // redirect that the Client follows. 87 // 88 // If Jar is nil, cookies are only sent if they are explicitly 89 // set on the Request. 90 Jar CookieJar 91 92 // Timeout specifies a time limit for requests made by this 93 // Client. The timeout includes connection time, any 94 // redirects, and reading the response body. The timer remains 95 // running after Get, Head, Post, or Do return and will 96 // interrupt reading of the Response.Body. 97 // 98 // A Timeout of zero means no timeout. 99 // 100 // The Client cancels requests to the underlying Transport 101 // as if the Request's Context ended. 102 // 103 // For compatibility, the Client will also use the deprecated 104 // CancelRequest method on Transport if found. New 105 // RoundTripper implementations should use the Request's Context 106 // for cancellation instead of implementing CancelRequest. 107 Timeout time.Duration 108 } 109 110 // DefaultClient is the default Client and is used by Get, Head, and Post. 111 var DefaultClient = &Client{} 112 113 // RoundTripper is an interface representing the ability to execute a 114 // single HTTP transaction, obtaining the Response for a given Request. 115 // 116 // A RoundTripper must be safe for concurrent use by multiple 117 // goroutines. 118 type RoundTripper interface { 119 // RoundTrip executes a single HTTP transaction, returning 120 // a Response for the provided Request. 121 // 122 // RoundTrip should not attempt to interpret the response. In 123 // particular, RoundTrip must return err == nil if it obtained 124 // a response, regardless of the response's HTTP status code. 125 // A non-nil err should be reserved for failure to obtain a 126 // response. Similarly, RoundTrip should not attempt to 127 // handle higher-level protocol details such as redirects, 128 // authentication, or cookies. 129 // 130 // RoundTrip should not modify the request, except for 131 // consuming and closing the Request's Body. RoundTrip may 132 // read fields of the request in a separate goroutine. Callers 133 // should not mutate or reuse the request until the Response's 134 // Body has been closed. 135 // 136 // RoundTrip must always close the body, including on errors, 137 // but depending on the implementation may do so in a separate 138 // goroutine even after RoundTrip returns. This means that 139 // callers wanting to reuse the body for subsequent requests 140 // must arrange to wait for the Close call before doing so. 141 // 142 // The Request's URL and Header fields must be initialized. 143 RoundTrip(*Request) (*Response, error) 144 } 145 146 // refererForURL returns a referer without any authentication info or 147 // an empty string if lastReq scheme is https and newReq scheme is http. 148 func refererForURL(lastReq, newReq *url.URL) string { 149 // https://tools.ietf.org/html/rfc7231#section-5.5.2 150 // "Clients SHOULD NOT include a Referer header field in a 151 // (non-secure) HTTP request if the referring page was 152 // transferred with a secure protocol." 153 if lastReq.Scheme == "https" && newReq.Scheme == "http" { 154 return "" 155 } 156 referer := lastReq.String() 157 if lastReq.User != nil { 158 // This is not very efficient, but is the best we can 159 // do without: 160 // - introducing a new method on URL 161 // - creating a race condition 162 // - copying the URL struct manually, which would cause 163 // maintenance problems down the line 164 auth := lastReq.User.String() + "@" 165 referer = strings.Replace(referer, auth, "", 1) 166 } 167 return referer 168 } 169 170 // didTimeout is non-nil only if err != nil. 171 func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 172 if c.Jar != nil { 173 for _, cookie := range c.Jar.Cookies(req.URL) { 174 req.AddCookie(cookie) 175 } 176 } 177 resp, didTimeout, err = send(req, c.transport(), deadline) 178 if err != nil { 179 return nil, didTimeout, err 180 } 181 if c.Jar != nil { 182 if rc := resp.Cookies(); len(rc) > 0 { 183 c.Jar.SetCookies(req.URL, rc) 184 } 185 } 186 return resp, nil, nil 187 } 188 189 func (c *Client) deadline() time.Time { 190 if c.Timeout > 0 { 191 return time.Now().Add(c.Timeout) 192 } 193 return time.Time{} 194 } 195 196 func (c *Client) transport() RoundTripper { 197 if c.Transport != nil { 198 return c.Transport 199 } 200 return DefaultTransport 201 } 202 203 // send issues an HTTP request. 204 // Caller should close resp.Body when done reading from it. 205 func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 206 req := ireq // req is either the original request, or a modified fork 207 208 if rt == nil { 209 req.closeBody() 210 return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") 211 } 212 213 if req.URL == nil { 214 req.closeBody() 215 return nil, alwaysFalse, errors.New("http: nil Request.URL") 216 } 217 218 if req.RequestURI != "" { 219 req.closeBody() 220 return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") 221 } 222 223 // forkReq forks req into a shallow clone of ireq the first 224 // time it's called. 225 forkReq := func() { 226 if ireq == req { 227 req = new(Request) 228 *req = *ireq // shallow clone 229 } 230 } 231 232 // Most the callers of send (Get, Post, et al) don't need 233 // Headers, leaving it uninitialized. We guarantee to the 234 // Transport that this has been initialized, though. 235 if req.Header == nil { 236 forkReq() 237 req.Header = make(Header) 238 } 239 240 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { 241 username := u.Username() 242 password, _ := u.Password() 243 forkReq() 244 req.Header = cloneOrMakeHeader(ireq.Header) 245 req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 246 } 247 248 if !deadline.IsZero() { 249 forkReq() 250 } 251 stopTimer, didTimeout := setRequestCancel(req, rt, deadline) 252 253 resp, err = rt.RoundTrip(req) 254 if err != nil { 255 stopTimer() 256 if resp != nil { 257 log.Printf("RoundTripper returned a response & error; ignoring response") 258 } 259 if tlsErr, ok := err.(tls.RecordHeaderError); ok { 260 // If we get a bad TLS record header, check to see if the 261 // response looks like HTTP and give a more helpful error. 262 // See golang.org/issue/11111. 263 if string(tlsErr.RecordHeader[:]) == "HTTP/" { 264 err = errors.New("http: server gave HTTP response to HTTPS client") 265 } 266 } 267 return nil, didTimeout, err 268 } 269 if resp == nil { 270 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt) 271 } 272 if resp.Body == nil { 273 // The documentation on the Body field says “The http Client and Transport 274 // guarantee that Body is always non-nil, even on responses without a body 275 // or responses with a zero-length body.” Unfortunately, we didn't document 276 // that same constraint for arbitrary RoundTripper implementations, and 277 // RoundTripper implementations in the wild (mostly in tests) assume that 278 // they can use a nil Body to mean an empty one (similar to Request.Body). 279 // (See https://golang.org/issue/38095.) 280 // 281 // If the ContentLength allows the Body to be empty, fill in an empty one 282 // here to ensure that it is non-nil. 283 if resp.ContentLength > 0 && req.Method != "HEAD" { 284 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength) 285 } 286 resp.Body = io.NopCloser(strings.NewReader("")) 287 } 288 if !deadline.IsZero() { 289 resp.Body = &cancelTimerBody{ 290 stop: stopTimer, 291 rc: resp.Body, 292 reqDidTimeout: didTimeout, 293 } 294 } 295 return resp, nil, nil 296 } 297 298 // timeBeforeContextDeadline reports whether the non-zero Time t is 299 // before ctx's deadline, if any. If ctx does not have a deadline, it 300 // always reports true (the deadline is considered infinite). 301 func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { 302 d, ok := ctx.Deadline() 303 if !ok { 304 return true 305 } 306 return t.Before(d) 307 } 308 309 // knownRoundTripperImpl reports whether rt is a RoundTripper that's 310 // maintained by the Go team and known to implement the latest 311 // optional semantics (notably contexts). The Request is used 312 // to check whether this particular request is using an alternate protocol, 313 // in which case we need to check the RoundTripper for that protocol. 314 func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { 315 switch t := rt.(type) { 316 case *Transport: 317 if altRT := t.alternateRoundTripper(req); altRT != nil { 318 return knownRoundTripperImpl(altRT, req) 319 } 320 return true 321 case *http2Transport, http2noDialH2RoundTripper: 322 return true 323 } 324 // There's a very minor chance of a false positive with this. 325 // Instead of detecting our golang.org/x/net/http2.Transport, 326 // it might detect a Transport type in a different http2 327 // package. But I know of none, and the only problem would be 328 // some temporarily leaked goroutines if the transport didn't 329 // support contexts. So this is a good enough heuristic: 330 if reflect.TypeOf(rt).String() == "*http2.Transport" { 331 return true 332 } 333 return false 334 } 335 336 // setRequestCancel sets req.Cancel and adds a deadline context to req 337 // if deadline is non-zero. The RoundTripper's type is used to 338 // determine whether the legacy CancelRequest behavior should be used. 339 // 340 // As background, there are three ways to cancel a request: 341 // First was Transport.CancelRequest. (deprecated) 342 // Second was Request.Cancel. 343 // Third was Request.Context. 344 // This function populates the second and third, and uses the first if it really needs to. 345 func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { 346 if deadline.IsZero() { 347 return nop, alwaysFalse 348 } 349 knownTransport := knownRoundTripperImpl(rt, req) 350 oldCtx := req.Context() 351 352 if req.Cancel == nil && knownTransport { 353 // If they already had a Request.Context that's 354 // expiring sooner, do nothing: 355 if !timeBeforeContextDeadline(deadline, oldCtx) { 356 return nop, alwaysFalse 357 } 358 359 var cancelCtx func() 360 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 361 return cancelCtx, func() bool { return time.Now().After(deadline) } 362 } 363 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any 364 365 var cancelCtx func() 366 if timeBeforeContextDeadline(deadline, oldCtx) { 367 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 368 } 369 370 cancel := make(chan struct{}) 371 req.Cancel = cancel 372 373 doCancel := func() { 374 // The second way in the func comment above: 375 close(cancel) 376 // The first way, used only for RoundTripper 377 // implementations written before Go 1.5 or Go 1.6. 378 type canceler interface{ CancelRequest(*Request) } 379 if v, ok := rt.(canceler); ok { 380 v.CancelRequest(req) 381 } 382 } 383 384 stopTimerCh := make(chan struct{}) 385 var once sync.Once 386 stopTimer = func() { 387 once.Do(func() { 388 close(stopTimerCh) 389 if cancelCtx != nil { 390 cancelCtx() 391 } 392 }) 393 } 394 395 timer := time.NewTimer(time.Until(deadline)) 396 var timedOut atomic.Bool 397 398 go func() { 399 select { 400 case <-initialReqCancel: 401 doCancel() 402 timer.Stop() 403 case <-timer.C: 404 timedOut.Store(true) 405 doCancel() 406 case <-stopTimerCh: 407 timer.Stop() 408 } 409 }() 410 411 return stopTimer, timedOut.Load 412 } 413 414 // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt 415 // "To receive authorization, the client sends the userid and password, 416 // separated by a single colon (":") character, within a base64 417 // encoded string in the credentials." 418 // It is not meant to be urlencoded. 419 func basicAuth(username, password string) string { 420 auth := username + ":" + password 421 return base64.StdEncoding.EncodeToString([]byte(auth)) 422 } 423 424 // Get issues a GET to the specified URL. If the response is one of 425 // the following redirect codes, Get follows the redirect, up to a 426 // maximum of 10 redirects: 427 // 428 // 301 (Moved Permanently) 429 // 302 (Found) 430 // 303 (See Other) 431 // 307 (Temporary Redirect) 432 // 308 (Permanent Redirect) 433 // 434 // An error is returned if there were too many redirects or if there 435 // was an HTTP protocol error. A non-2xx response doesn't cause an 436 // error. Any returned error will be of type *url.Error. The url.Error 437 // value's Timeout method will report true if the request timed out. 438 // 439 // When err is nil, resp always contains a non-nil resp.Body. 440 // Caller should close resp.Body when done reading from it. 441 // 442 // Get is a wrapper around DefaultClient.Get. 443 // 444 // To make a request with custom headers, use NewRequest and 445 // DefaultClient.Do. 446 // 447 // To make a request with a specified context.Context, use NewRequestWithContext 448 // and DefaultClient.Do. 449 func Get(url string) (resp *Response, err error) { 450 return DefaultClient.Get(url) 451 } 452 453 // Get issues a GET to the specified URL. If the response is one of the 454 // following redirect codes, Get follows the redirect after calling the 455 // Client's CheckRedirect function: 456 // 457 // 301 (Moved Permanently) 458 // 302 (Found) 459 // 303 (See Other) 460 // 307 (Temporary Redirect) 461 // 308 (Permanent Redirect) 462 // 463 // An error is returned if the Client's CheckRedirect function fails 464 // or if there was an HTTP protocol error. A non-2xx response doesn't 465 // cause an error. Any returned error will be of type *url.Error. The 466 // url.Error value's Timeout method will report true if the request 467 // timed out. 468 // 469 // When err is nil, resp always contains a non-nil resp.Body. 470 // Caller should close resp.Body when done reading from it. 471 // 472 // To make a request with custom headers, use NewRequest and Client.Do. 473 // 474 // To make a request with a specified context.Context, use NewRequestWithContext 475 // and Client.Do. 476 func (c *Client) Get(url string) (resp *Response, err error) { 477 req, err := NewRequest("GET", url, nil) 478 if err != nil { 479 return nil, err 480 } 481 return c.Do(req) 482 } 483 484 func alwaysFalse() bool { return false } 485 486 // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to 487 // control how redirects are processed. If returned, the next request 488 // is not sent and the most recent response is returned with its body 489 // unclosed. 490 var ErrUseLastResponse = errors.New("net/http: use last response") 491 492 // checkRedirect calls either the user's configured CheckRedirect 493 // function, or the default. 494 func (c *Client) checkRedirect(req *Request, via []*Request) error { 495 fn := c.CheckRedirect 496 if fn == nil { 497 fn = defaultCheckRedirect 498 } 499 return fn(req, via) 500 } 501 502 // redirectBehavior describes what should happen when the 503 // client encounters a 3xx status code from the server. 504 func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) { 505 switch resp.StatusCode { 506 case 301, 302, 303: 507 redirectMethod = reqMethod 508 shouldRedirect = true 509 includeBody = false 510 511 // RFC 2616 allowed automatic redirection only with GET and 512 // HEAD requests. RFC 7231 lifts this restriction, but we still 513 // restrict other methods to GET to maintain compatibility. 514 // See Issue 18570. 515 if reqMethod != "GET" && reqMethod != "HEAD" { 516 redirectMethod = "GET" 517 } 518 case 307, 308: 519 redirectMethod = reqMethod 520 shouldRedirect = true 521 includeBody = true 522 523 if ireq.GetBody == nil && ireq.outgoingLength() != 0 { 524 // We had a request body, and 307/308 require 525 // re-sending it, but GetBody is not defined. So just 526 // return this response to the user instead of an 527 // error, like we did in Go 1.7 and earlier. 528 shouldRedirect = false 529 } 530 } 531 return redirectMethod, shouldRedirect, includeBody 532 } 533 534 // urlErrorOp returns the (*url.Error).Op value to use for the 535 // provided (*Request).Method value. 536 func urlErrorOp(method string) string { 537 if method == "" { 538 return "Get" 539 } 540 if lowerMethod, ok := ascii.ToLower(method); ok { 541 return method[:1] + lowerMethod[1:] 542 } 543 return method 544 } 545 546 // Do sends an HTTP request and returns an HTTP response, following 547 // policy (such as redirects, cookies, auth) as configured on the 548 // client. 549 // 550 // An error is returned if caused by client policy (such as 551 // CheckRedirect), or failure to speak HTTP (such as a network 552 // connectivity problem). A non-2xx status code doesn't cause an 553 // error. 554 // 555 // If the returned error is nil, the Response will contain a non-nil 556 // Body which the user is expected to close. If the Body is not both 557 // read to EOF and closed, the Client's underlying RoundTripper 558 // (typically Transport) may not be able to re-use a persistent TCP 559 // connection to the server for a subsequent "keep-alive" request. 560 // 561 // The request Body, if non-nil, will be closed by the underlying 562 // Transport, even on errors. 563 // 564 // On error, any Response can be ignored. A non-nil Response with a 565 // non-nil error only occurs when CheckRedirect fails, and even then 566 // the returned Response.Body is already closed. 567 // 568 // Generally Get, Post, or PostForm will be used instead of Do. 569 // 570 // If the server replies with a redirect, the Client first uses the 571 // CheckRedirect function to determine whether the redirect should be 572 // followed. If permitted, a 301, 302, or 303 redirect causes 573 // subsequent requests to use HTTP method GET 574 // (or HEAD if the original request was HEAD), with no body. 575 // A 307 or 308 redirect preserves the original HTTP method and body, 576 // provided that the Request.GetBody function is defined. 577 // The NewRequest function automatically sets GetBody for common 578 // standard library body types. 579 // 580 // Any returned error will be of type *url.Error. The url.Error 581 // value's Timeout method will report true if the request timed out. 582 func (c *Client) Do(req *Request) (*Response, error) { 583 return c.do(req) 584 } 585 586 var testHookClientDoResult func(retres *Response, reterr error) 587 588 func (c *Client) do(req *Request) (retres *Response, reterr error) { 589 if testHookClientDoResult != nil { 590 defer func() { testHookClientDoResult(retres, reterr) }() 591 } 592 if req.URL == nil { 593 req.closeBody() 594 return nil, &url.Error{ 595 Op: urlErrorOp(req.Method), 596 Err: errors.New("http: nil Request.URL"), 597 } 598 } 599 600 var ( 601 deadline = c.deadline() 602 reqs []*Request 603 resp *Response 604 copyHeaders = c.makeHeadersCopier(req) 605 reqBodyClosed = false // have we closed the current req.Body? 606 607 // Redirect behavior: 608 redirectMethod string 609 includeBody bool 610 ) 611 uerr := func(err error) error { 612 // the body may have been closed already by c.send() 613 if !reqBodyClosed { 614 req.closeBody() 615 } 616 var urlStr string 617 if resp != nil && resp.Request != nil { 618 urlStr = stripPassword(resp.Request.URL) 619 } else { 620 urlStr = stripPassword(req.URL) 621 } 622 return &url.Error{ 623 Op: urlErrorOp(reqs[0].Method), 624 URL: urlStr, 625 Err: err, 626 } 627 } 628 for { 629 // For all but the first request, create the next 630 // request hop and replace req. 631 if len(reqs) > 0 { 632 loc := resp.Header.Get("Location") 633 if loc == "" { 634 // While most 3xx responses include a Location, it is not 635 // required and 3xx responses without a Location have been 636 // observed in the wild. See issues #17773 and #49281. 637 return resp, nil 638 } 639 u, err := req.URL.Parse(loc) 640 if err != nil { 641 resp.closeBody() 642 return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) 643 } 644 host := "" 645 if req.Host != "" && req.Host != req.URL.Host { 646 // If the caller specified a custom Host header and the 647 // redirect location is relative, preserve the Host header 648 // through the redirect. See issue #22233. 649 if u, _ := url.Parse(loc); u != nil && !u.IsAbs() { 650 host = req.Host 651 } 652 } 653 ireq := reqs[0] 654 req = &Request{ 655 Method: redirectMethod, 656 Response: resp, 657 URL: u, 658 Header: make(Header), 659 Host: host, 660 Cancel: ireq.Cancel, 661 ctx: ireq.ctx, 662 } 663 if includeBody && ireq.GetBody != nil { 664 req.Body, err = ireq.GetBody() 665 if err != nil { 666 resp.closeBody() 667 return nil, uerr(err) 668 } 669 req.ContentLength = ireq.ContentLength 670 } 671 672 // Copy original headers before setting the Referer, 673 // in case the user set Referer on their first request. 674 // If they really want to override, they can do it in 675 // their CheckRedirect func. 676 copyHeaders(req) 677 678 // Add the Referer header from the most recent 679 // request URL to the new one, if it's not https->http: 680 if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" { 681 req.Header.Set("Referer", ref) 682 } 683 err = c.checkRedirect(req, reqs) 684 685 // Sentinel error to let users select the 686 // previous response, without closing its 687 // body. See Issue 10069. 688 if err == ErrUseLastResponse { 689 return resp, nil 690 } 691 692 // Close the previous response's body. But 693 // read at least some of the body so if it's 694 // small the underlying TCP connection will be 695 // re-used. No need to check for errors: if it 696 // fails, the Transport won't reuse it anyway. 697 const maxBodySlurpSize = 2 << 10 698 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { 699 io.CopyN(io.Discard, resp.Body, maxBodySlurpSize) 700 } 701 resp.Body.Close() 702 703 if err != nil { 704 // Special case for Go 1 compatibility: return both the response 705 // and an error if the CheckRedirect function failed. 706 // See https://golang.org/issue/3795 707 // The resp.Body has already been closed. 708 ue := uerr(err) 709 ue.(*url.Error).URL = loc 710 return resp, ue 711 } 712 } 713 714 reqs = append(reqs, req) 715 var err error 716 var didTimeout func() bool 717 if resp, didTimeout, err = c.send(req, deadline); err != nil { 718 // c.send() always closes req.Body 719 reqBodyClosed = true 720 if !deadline.IsZero() && didTimeout() { 721 err = &httpError{ 722 err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", 723 timeout: true, 724 } 725 } 726 return nil, uerr(err) 727 } 728 729 var shouldRedirect bool 730 redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0]) 731 if !shouldRedirect { 732 return resp, nil 733 } 734 735 req.closeBody() 736 } 737 } 738 739 // makeHeadersCopier makes a function that copies headers from the 740 // initial Request, ireq. For every redirect, this function must be called 741 // so that it can copy headers into the upcoming Request. 742 func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { 743 // The headers to copy are from the very initial request. 744 // We use a closured callback to keep a reference to these original headers. 745 var ( 746 ireqhdr = cloneOrMakeHeader(ireq.Header) 747 icookies map[string][]*Cookie 748 ) 749 if c.Jar != nil && ireq.Header.Get("Cookie") != "" { 750 icookies = make(map[string][]*Cookie) 751 for _, c := range ireq.Cookies() { 752 icookies[c.Name] = append(icookies[c.Name], c) 753 } 754 } 755 756 preq := ireq // The previous request 757 return func(req *Request) { 758 // If Jar is present and there was some initial cookies provided 759 // via the request header, then we may need to alter the initial 760 // cookies as we follow redirects since each redirect may end up 761 // modifying a pre-existing cookie. 762 // 763 // Since cookies already set in the request header do not contain 764 // information about the original domain and path, the logic below 765 // assumes any new set cookies override the original cookie 766 // regardless of domain or path. 767 // 768 // See https://golang.org/issue/17494 769 if c.Jar != nil && icookies != nil { 770 var changed bool 771 resp := req.Response // The response that caused the upcoming redirect 772 for _, c := range resp.Cookies() { 773 if _, ok := icookies[c.Name]; ok { 774 delete(icookies, c.Name) 775 changed = true 776 } 777 } 778 if changed { 779 ireqhdr.Del("Cookie") 780 var ss []string 781 for _, cs := range icookies { 782 for _, c := range cs { 783 ss = append(ss, c.Name+"="+c.Value) 784 } 785 } 786 sort.Strings(ss) // Ensure deterministic headers 787 ireqhdr.Set("Cookie", strings.Join(ss, "; ")) 788 } 789 } 790 791 // Copy the initial request's Header Values 792 // (at least the safe ones). 793 for k, vv := range ireqhdr { 794 if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) { 795 req.Header[k] = vv 796 } 797 } 798 799 preq = req // Update previous Request with the current request 800 } 801 } 802 803 func defaultCheckRedirect(req *Request, via []*Request) error { 804 if len(via) >= 10 { 805 return errors.New("stopped after 10 redirects") 806 } 807 return nil 808 } 809 810 // Post issues a POST to the specified URL. 811 // 812 // Caller should close resp.Body when done reading from it. 813 // 814 // If the provided body is an io.Closer, it is closed after the 815 // request. 816 // 817 // Post is a wrapper around DefaultClient.Post. 818 // 819 // To set custom headers, use NewRequest and DefaultClient.Do. 820 // 821 // See the Client.Do method documentation for details on how redirects 822 // are handled. 823 // 824 // To make a request with a specified context.Context, use NewRequestWithContext 825 // and DefaultClient.Do. 826 func Post(url, contentType string, body io.Reader) (resp *Response, err error) { 827 return DefaultClient.Post(url, contentType, body) 828 } 829 830 // Post issues a POST to the specified URL. 831 // 832 // Caller should close resp.Body when done reading from it. 833 // 834 // If the provided body is an io.Closer, it is closed after the 835 // request. 836 // 837 // To set custom headers, use NewRequest and Client.Do. 838 // 839 // To make a request with a specified context.Context, use NewRequestWithContext 840 // and Client.Do. 841 // 842 // See the Client.Do method documentation for details on how redirects 843 // are handled. 844 func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { 845 req, err := NewRequest("POST", url, body) 846 if err != nil { 847 return nil, err 848 } 849 req.Header.Set("Content-Type", contentType) 850 return c.Do(req) 851 } 852 853 // PostForm issues a POST to the specified URL, with data's keys and 854 // Values URL-encoded as the request body. 855 // 856 // The Content-Type header is set to application/x-www-form-urlencoded. 857 // To set other headers, use NewRequest and DefaultClient.Do. 858 // 859 // When err is nil, resp always contains a non-nil resp.Body. 860 // Caller should close resp.Body when done reading from it. 861 // 862 // PostForm is a wrapper around DefaultClient.PostForm. 863 // 864 // See the Client.Do method documentation for details on how redirects 865 // are handled. 866 // 867 // To make a request with a specified context.Context, use NewRequestWithContext 868 // and DefaultClient.Do. 869 func PostForm(url string, data url.Values) (resp *Response, err error) { 870 return DefaultClient.PostForm(url, data) 871 } 872 873 // PostForm issues a POST to the specified URL, 874 // with data's keys and Values URL-encoded as the request body. 875 // 876 // The Content-Type header is set to application/x-www-form-urlencoded. 877 // To set other headers, use NewRequest and Client.Do. 878 // 879 // When err is nil, resp always contains a non-nil resp.Body. 880 // Caller should close resp.Body when done reading from it. 881 // 882 // See the Client.Do method documentation for details on how redirects 883 // are handled. 884 // 885 // To make a request with a specified context.Context, use NewRequestWithContext 886 // and Client.Do. 887 func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { 888 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 889 } 890 891 // Head issues a HEAD to the specified URL. If the response is one of 892 // the following redirect codes, Head follows the redirect, up to a 893 // maximum of 10 redirects: 894 // 895 // 301 (Moved Permanently) 896 // 302 (Found) 897 // 303 (See Other) 898 // 307 (Temporary Redirect) 899 // 308 (Permanent Redirect) 900 // 901 // Head is a wrapper around DefaultClient.Head. 902 // 903 // To make a request with a specified context.Context, use NewRequestWithContext 904 // and DefaultClient.Do. 905 func Head(url string) (resp *Response, err error) { 906 return DefaultClient.Head(url) 907 } 908 909 // Head issues a HEAD to the specified URL. If the response is one of the 910 // following redirect codes, Head follows the redirect after calling the 911 // Client's CheckRedirect function: 912 // 913 // 301 (Moved Permanently) 914 // 302 (Found) 915 // 303 (See Other) 916 // 307 (Temporary Redirect) 917 // 308 (Permanent Redirect) 918 // 919 // To make a request with a specified context.Context, use NewRequestWithContext 920 // and Client.Do. 921 func (c *Client) Head(url string) (resp *Response, err error) { 922 req, err := NewRequest("HEAD", url, nil) 923 if err != nil { 924 return nil, err 925 } 926 return c.Do(req) 927 } 928 929 // CloseIdleConnections closes any connections on its Transport which 930 // were previously connected from previous requests but are now 931 // sitting idle in a "keep-alive" state. It does not interrupt any 932 // connections currently in use. 933 // 934 // If the Client's Transport does not have a CloseIdleConnections method 935 // then this method does nothing. 936 func (c *Client) CloseIdleConnections() { 937 type closeIdler interface { 938 CloseIdleConnections() 939 } 940 if tr, ok := c.transport().(closeIdler); ok { 941 tr.CloseIdleConnections() 942 } 943 } 944 945 // cancelTimerBody is an io.ReadCloser that wraps rc with two features: 946 // 1. On Read error or close, the stop func is called. 947 // 2. On Read failure, if reqDidTimeout is true, the error is wrapped and 948 // marked as net.Error that hit its timeout. 949 type cancelTimerBody struct { 950 stop func() // stops the time.Timer waiting to cancel the request 951 rc io.ReadCloser 952 reqDidTimeout func() bool 953 } 954 955 func (b *cancelTimerBody) Read(p []byte) (n int, err error) { 956 n, err = b.rc.Read(p) 957 if err == nil { 958 return n, nil 959 } 960 if err == io.EOF { 961 return n, err 962 } 963 if b.reqDidTimeout() { 964 err = &httpError{ 965 err: err.Error() + " (Client.Timeout or context cancellation while reading body)", 966 timeout: true, 967 } 968 } 969 return n, err 970 } 971 972 func (b *cancelTimerBody) Close() error { 973 err := b.rc.Close() 974 b.stop() 975 return err 976 } 977 978 func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { 979 switch CanonicalHeaderKey(headerKey) { 980 case "Authorization", "Www-Authenticate", "Cookie", "Cookie2": 981 // Permit sending auth/cookie headers from "foo.com" 982 // to "sub.foo.com". 983 984 // Note that we don't send all cookies to subdomains 985 // automatically. This function is only used for 986 // Cookies set explicitly on the initial outgoing 987 // client request. Cookies automatically added via the 988 // CookieJar mechanism continue to follow each 989 // cookie's scope as set by Set-Cookie. But for 990 // outgoing requests with the Cookie header set 991 // directly, we don't know their scope, so we assume 992 // it's for *.domain.com. 993 994 ihost := idnaASCIIFromURL(initial) 995 dhost := idnaASCIIFromURL(dest) 996 return isDomainOrSubdomain(dhost, ihost) 997 } 998 // All other headers are copied: 999 return true 1000 } 1001 1002 // isDomainOrSubdomain reports whether sub is a subdomain (or exact 1003 // match) of the parent domain. 1004 // 1005 // Both domains must already be in canonical form. 1006 func isDomainOrSubdomain(sub, parent string) bool { 1007 if sub == parent { 1008 return true 1009 } 1010 // If sub is "foo.example.com" and parent is "example.com", 1011 // that means sub must end in "."+parent. 1012 // Do it without allocating. 1013 if !strings.HasSuffix(sub, parent) { 1014 return false 1015 } 1016 return sub[len(sub)-len(parent)-1] == '.' 1017 } 1018 1019 func stripPassword(u *url.URL) string { 1020 _, passSet := u.User.Password() 1021 if passSet { 1022 return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) 1023 } 1024 return u.String() 1025 }