github.com/goproxy0/go@v0.0.0-20171111080102-49cc0c489d2c/src/net/http/transport.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client implementation. See RFC 2616. 6 // 7 // This is the low-level Transport implementation of RoundTripper. 8 // The high-level interface is in client.go. 9 10 package http 11 12 import ( 13 "bufio" 14 "compress/gzip" 15 "container/list" 16 "context" 17 "crypto/tls" 18 "errors" 19 "fmt" 20 "io" 21 "log" 22 "net" 23 "net/http/httptrace" 24 "net/url" 25 "os" 26 "strings" 27 "sync" 28 "sync/atomic" 29 "time" 30 31 "golang_org/x/net/lex/httplex" 32 "golang_org/x/net/proxy" 33 ) 34 35 // DefaultTransport is the default implementation of Transport and is 36 // used by DefaultClient. It establishes network connections as needed 37 // and caches them for reuse by subsequent calls. It uses HTTP proxies 38 // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and 39 // $no_proxy) environment variables. 40 var DefaultTransport RoundTripper = &Transport{ 41 Proxy: ProxyFromEnvironment, 42 DialContext: (&net.Dialer{ 43 Timeout: 30 * time.Second, 44 KeepAlive: 30 * time.Second, 45 DualStack: true, 46 }).DialContext, 47 MaxIdleConns: 100, 48 IdleConnTimeout: 90 * time.Second, 49 TLSHandshakeTimeout: 10 * time.Second, 50 ExpectContinueTimeout: 1 * time.Second, 51 } 52 53 // DefaultMaxIdleConnsPerHost is the default value of Transport's 54 // MaxIdleConnsPerHost. 55 const DefaultMaxIdleConnsPerHost = 2 56 57 // Transport is an implementation of RoundTripper that supports HTTP, 58 // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 59 // 60 // By default, Transport caches connections for future re-use. 61 // This may leave many open connections when accessing many hosts. 62 // This behavior can be managed using Transport's CloseIdleConnections method 63 // and the MaxIdleConnsPerHost and DisableKeepAlives fields. 64 // 65 // Transports should be reused instead of created as needed. 66 // Transports are safe for concurrent use by multiple goroutines. 67 // 68 // A Transport is a low-level primitive for making HTTP and HTTPS requests. 69 // For high-level functionality, such as cookies and redirects, see Client. 70 // 71 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 72 // for HTTPS URLs, depending on whether the server supports HTTP/2, 73 // and how the Transport is configured. The DefaultTransport supports HTTP/2. 74 // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 75 // and call ConfigureTransport. See the package docs for more about HTTP/2. 76 type Transport struct { 77 idleMu sync.Mutex 78 wantIdle bool // user has requested to close all idle conns 79 idleConn map[connectMethodKey][]*persistConn // most recently used at end 80 idleConnCh map[connectMethodKey]chan *persistConn 81 idleLRU connLRU 82 83 reqMu sync.Mutex 84 reqCanceler map[*Request]func(error) 85 86 altMu sync.Mutex // guards changing altProto only 87 altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme 88 89 // Proxy specifies a function to return a proxy for a given 90 // Request. If the function returns a non-nil error, the 91 // request is aborted with the provided error. 92 // 93 // The proxy type is determined by the URL scheme. "http" 94 // and "socks5" are supported. If the scheme is empty, 95 // "http" is assumed. 96 // 97 // If Proxy is nil or returns a nil *URL, no proxy is used. 98 Proxy func(*Request) (*url.URL, error) 99 100 // GetConnectMethodAddr specifies a function to return a addr string for ConnectMethod 101 GetConnectMethodAddr func(addr string) string 102 103 // DialContext specifies the dial function for creating unencrypted TCP connections. 104 // If DialContext is nil (and the deprecated Dial below is also nil), 105 // then the transport dials using package net. 106 DialContext func(ctx context.Context, network, addr string) (net.Conn, error) 107 108 // Dial specifies the dial function for creating unencrypted TCP connections. 109 // 110 // Deprecated: Use DialContext instead, which allows the transport 111 // to cancel dials as soon as they are no longer needed. 112 // If both are set, DialContext takes priority. 113 Dial func(network, addr string) (net.Conn, error) 114 115 // DialTLS specifies an optional dial function for creating 116 // TLS connections for non-proxied HTTPS requests. 117 // 118 // If DialTLS is nil, Dial and TLSClientConfig are used. 119 // 120 // If DialTLS is set, the Dial hook is not used for HTTPS 121 // requests and the TLSClientConfig and TLSHandshakeTimeout 122 // are ignored. The returned net.Conn is assumed to already be 123 // past the TLS handshake. 124 DialTLS func(network, addr string) (net.Conn, error) 125 126 // TLSClientConfig specifies the TLS configuration to use with 127 // tls.Client. 128 // If nil, the default configuration is used. 129 // If non-nil, HTTP/2 support may not be enabled by default. 130 TLSClientConfig *tls.Config 131 132 // TLSHandshakeTimeout specifies the maximum amount of time waiting to 133 // wait for a TLS handshake. Zero means no timeout. 134 TLSHandshakeTimeout time.Duration 135 136 // DisableKeepAlives, if true, prevents re-use of TCP connections 137 // between different HTTP requests. 138 DisableKeepAlives bool 139 140 // DisableCompression, if true, prevents the Transport from 141 // requesting compression with an "Accept-Encoding: gzip" 142 // request header when the Request contains no existing 143 // Accept-Encoding value. If the Transport requests gzip on 144 // its own and gets a gzipped response, it's transparently 145 // decoded in the Response.Body. However, if the user 146 // explicitly requested gzip it is not automatically 147 // uncompressed. 148 DisableCompression bool 149 150 // MaxIdleConns controls the maximum number of idle (keep-alive) 151 // connections across all hosts. Zero means no limit. 152 MaxIdleConns int 153 154 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 155 // (keep-alive) connections to keep per-host. If zero, 156 // DefaultMaxIdleConnsPerHost is used. 157 MaxIdleConnsPerHost int 158 159 // IdleConnTimeout is the maximum amount of time an idle 160 // (keep-alive) connection will remain idle before closing 161 // itself. 162 // Zero means no limit. 163 IdleConnTimeout time.Duration 164 165 // ResponseHeaderTimeout, if non-zero, specifies the amount of 166 // time to wait for a server's response headers after fully 167 // writing the request (including its body, if any). This 168 // time does not include the time to read the response body. 169 ResponseHeaderTimeout time.Duration 170 171 // ExpectContinueTimeout, if non-zero, specifies the amount of 172 // time to wait for a server's first response headers after fully 173 // writing the request headers if the request has an 174 // "Expect: 100-continue" header. Zero means no timeout and 175 // causes the body to be sent immediately, without 176 // waiting for the server to approve. 177 // This time does not include the time to send the request header. 178 ExpectContinueTimeout time.Duration 179 180 // TLSNextProto specifies how the Transport switches to an 181 // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN 182 // protocol negotiation. If Transport dials an TLS connection 183 // with a non-empty protocol name and TLSNextProto contains a 184 // map entry for that key (such as "h2"), then the func is 185 // called with the request's authority (such as "example.com" 186 // or "example.com:1234") and the TLS connection. The function 187 // must return a RoundTripper that then handles the request. 188 // If TLSNextProto is not nil, HTTP/2 support is not enabled 189 // automatically. 190 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 191 192 // ProxyConnectHeader optionally specifies headers to send to 193 // proxies during CONNECT requests. 194 ProxyConnectHeader Header 195 196 // MaxResponseHeaderBytes specifies a limit on how many 197 // response bytes are allowed in the server's response 198 // header. 199 // 200 // Zero means to use a default limit. 201 MaxResponseHeaderBytes int64 202 203 // nextProtoOnce guards initialization of TLSNextProto and 204 // h2transport (via onceSetNextProtoDefaults) 205 nextProtoOnce sync.Once 206 h2transport *http2Transport // non-nil if http2 wired up 207 208 // TODO: tunable on max per-host TCP dials in flight (Issue 13957) 209 } 210 211 // onceSetNextProtoDefaults initializes TLSNextProto. 212 // It must be called via t.nextProtoOnce.Do. 213 func (t *Transport) onceSetNextProtoDefaults() { 214 if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") { 215 return 216 } 217 if t.TLSNextProto != nil { 218 // This is the documented way to disable http2 on a 219 // Transport. 220 return 221 } 222 if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil { 223 // Be conservative and don't automatically enable 224 // http2 if they've specified a custom TLS config or 225 // custom dialers. Let them opt-in themselves via 226 // http2.ConfigureTransport so we don't surprise them 227 // by modifying their tls.Config. Issue 14275. 228 return 229 } 230 t2, err := http2configureTransport(t) 231 if err != nil { 232 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 233 return 234 } 235 t.h2transport = t2 236 237 // Auto-configure the http2.Transport's MaxHeaderListSize from 238 // the http.Transport's MaxResponseHeaderBytes. They don't 239 // exactly mean the same thing, but they're close. 240 // 241 // TODO: also add this to x/net/http2.Configure Transport, behind 242 // a +build go1.7 build tag: 243 if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { 244 const h2max = 1<<32 - 1 245 if limit1 >= h2max { 246 t2.MaxHeaderListSize = h2max 247 } else { 248 t2.MaxHeaderListSize = uint32(limit1) 249 } 250 } 251 } 252 253 // ProxyFromEnvironment returns the URL of the proxy to use for a 254 // given request, as indicated by the environment variables 255 // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 256 // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https 257 // requests. 258 // 259 // The environment values may be either a complete URL or a 260 // "host[:port]", in which case the "http" scheme is assumed. 261 // An error is returned if the value is a different form. 262 // 263 // A nil URL and nil error are returned if no proxy is defined in the 264 // environment, or a proxy should not be used for the given request, 265 // as defined by NO_PROXY. 266 // 267 // As a special case, if req.URL.Host is "localhost" (with or without 268 // a port number), then a nil URL and nil error will be returned. 269 func ProxyFromEnvironment(req *Request) (*url.URL, error) { 270 var proxy string 271 if req.URL.Scheme == "https" { 272 proxy = httpsProxyEnv.Get() 273 } 274 if proxy == "" { 275 proxy = httpProxyEnv.Get() 276 if proxy != "" && os.Getenv("REQUEST_METHOD") != "" { 277 return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") 278 } 279 } 280 if proxy == "" { 281 return nil, nil 282 } 283 if !useProxy(canonicalAddr(req.URL)) { 284 return nil, nil 285 } 286 proxyURL, err := url.Parse(proxy) 287 if err != nil || 288 (proxyURL.Scheme != "http" && 289 proxyURL.Scheme != "https" && 290 proxyURL.Scheme != "socks5") { 291 // proxy was bogus. Try prepending "http://" to it and 292 // see if that parses correctly. If not, we fall 293 // through and complain about the original one. 294 if proxyURL, err := url.Parse("http://" + proxy); err == nil { 295 return proxyURL, nil 296 } 297 298 } 299 if err != nil { 300 return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) 301 } 302 return proxyURL, nil 303 } 304 305 // ProxyURL returns a proxy function (for use in a Transport) 306 // that always returns the same URL. 307 func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 308 return func(*Request) (*url.URL, error) { 309 return fixedURL, nil 310 } 311 } 312 313 // transportRequest is a wrapper around a *Request that adds 314 // optional extra headers to write and stores any error to return 315 // from roundTrip. 316 type transportRequest struct { 317 *Request // original request, not to be mutated 318 extra Header // extra headers to write, or nil 319 trace *httptrace.ClientTrace // optional 320 321 mu sync.Mutex // guards err 322 err error // first setError value for mapRoundTripError to consider 323 } 324 325 func (tr *transportRequest) extraHeaders() Header { 326 if tr.extra == nil { 327 tr.extra = make(Header) 328 } 329 return tr.extra 330 } 331 332 func (tr *transportRequest) setError(err error) { 333 tr.mu.Lock() 334 if tr.err == nil { 335 tr.err = err 336 } 337 tr.mu.Unlock() 338 } 339 340 // RoundTrip implements the RoundTripper interface. 341 // 342 // For higher-level HTTP client support (such as handling of cookies 343 // and redirects), see Get, Post, and the Client type. 344 func (t *Transport) RoundTrip(req *Request) (*Response, error) { 345 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 346 ctx := req.Context() 347 trace := httptrace.ContextClientTrace(ctx) 348 349 if req.URL == nil { 350 req.closeBody() 351 return nil, errors.New("http: nil Request.URL") 352 } 353 if req.Header == nil { 354 req.closeBody() 355 return nil, errors.New("http: nil Request.Header") 356 } 357 scheme := req.URL.Scheme 358 isHTTP := scheme == "http" || scheme == "https" 359 if isHTTP { 360 for k, vv := range req.Header { 361 if !httplex.ValidHeaderFieldName(k) { 362 return nil, fmt.Errorf("net/http: invalid header field name %q", k) 363 } 364 for _, v := range vv { 365 if !httplex.ValidHeaderFieldValue(v) { 366 return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) 367 } 368 } 369 } 370 } 371 372 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 373 if altRT := altProto[scheme]; altRT != nil { 374 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 375 return resp, err 376 } 377 } 378 if !isHTTP { 379 req.closeBody() 380 return nil, &badStringError{"unsupported protocol scheme", scheme} 381 } 382 if req.Method != "" && !validMethod(req.Method) { 383 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 384 } 385 if req.URL.Host == "" { 386 req.closeBody() 387 return nil, errors.New("http: no Host in request URL") 388 } 389 390 for { 391 // treq gets modified by roundTrip, so we need to recreate for each retry. 392 treq := &transportRequest{Request: req, trace: trace} 393 cm, err := t.connectMethodForRequest(treq) 394 if err != nil { 395 req.closeBody() 396 return nil, err 397 } 398 399 // Get the cached or newly-created connection to either the 400 // host (for http or https), the http proxy, or the http proxy 401 // pre-CONNECTed to https server. In any case, we'll be ready 402 // to send it requests. 403 pconn, err := t.getConn(treq, cm) 404 if err != nil { 405 t.setReqCanceler(req, nil) 406 req.closeBody() 407 return nil, err 408 } 409 410 var resp *Response 411 if pconn.alt != nil { 412 // HTTP/2 path. 413 t.setReqCanceler(req, nil) // not cancelable with CancelRequest 414 resp, err = pconn.alt.RoundTrip(req) 415 } else { 416 resp, err = pconn.roundTrip(treq) 417 } 418 if err == nil { 419 return resp, nil 420 } 421 if !pconn.shouldRetryRequest(req, err) { 422 // Issue 16465: return underlying net.Conn.Read error from peek, 423 // as we've historically done. 424 if e, ok := err.(transportReadFromServerError); ok { 425 err = e.err 426 } 427 return nil, err 428 } 429 testHookRoundTripRetried() 430 431 // Rewind the body if we're able to. (HTTP/2 does this itself so we only 432 // need to do it for HTTP/1.1 connections.) 433 if req.GetBody != nil && pconn.alt == nil { 434 newReq := *req 435 var err error 436 newReq.Body, err = req.GetBody() 437 if err != nil { 438 return nil, err 439 } 440 req = &newReq 441 } 442 } 443 } 444 445 // shouldRetryRequest reports whether we should retry sending a failed 446 // HTTP request on a new connection. The non-nil input error is the 447 // error from roundTrip. 448 func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { 449 if err == http2ErrNoCachedConn { 450 // Issue 16582: if the user started a bunch of 451 // requests at once, they can all pick the same conn 452 // and violate the server's max concurrent streams. 453 // Instead, match the HTTP/1 behavior for now and dial 454 // again to get a new TCP connection, rather than failing 455 // this request. 456 return true 457 } 458 if err == errMissingHost { 459 // User error. 460 return false 461 } 462 if !pc.isReused() { 463 // This was a fresh connection. There's no reason the server 464 // should've hung up on us. 465 // 466 // Also, if we retried now, we could loop forever 467 // creating new connections and retrying if the server 468 // is just hanging up on us because it doesn't like 469 // our request (as opposed to sending an error). 470 return false 471 } 472 if _, ok := err.(nothingWrittenError); ok { 473 // We never wrote anything, so it's safe to retry, if there's no body or we 474 // can "rewind" the body with GetBody. 475 return req.outgoingLength() == 0 || req.GetBody != nil 476 } 477 if !req.isReplayable() { 478 // Don't retry non-idempotent requests. 479 return false 480 } 481 if _, ok := err.(transportReadFromServerError); ok { 482 // We got some non-EOF net.Conn.Read failure reading 483 // the 1st response byte from the server. 484 return true 485 } 486 if err == errServerClosedIdle { 487 // The server replied with io.EOF while we were trying to 488 // read the response. Probably an unfortunately keep-alive 489 // timeout, just as the client was writing a request. 490 return true 491 } 492 return false // conservatively 493 } 494 495 // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 496 var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 497 498 // RegisterProtocol registers a new protocol with scheme. 499 // The Transport will pass requests using the given scheme to rt. 500 // It is rt's responsibility to simulate HTTP request semantics. 501 // 502 // RegisterProtocol can be used by other packages to provide 503 // implementations of protocol schemes like "ftp" or "file". 504 // 505 // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will 506 // handle the RoundTrip itself for that one request, as if the 507 // protocol were not registered. 508 func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 509 t.altMu.Lock() 510 defer t.altMu.Unlock() 511 oldMap, _ := t.altProto.Load().(map[string]RoundTripper) 512 if _, exists := oldMap[scheme]; exists { 513 panic("protocol " + scheme + " already registered") 514 } 515 newMap := make(map[string]RoundTripper) 516 for k, v := range oldMap { 517 newMap[k] = v 518 } 519 newMap[scheme] = rt 520 t.altProto.Store(newMap) 521 } 522 523 // CloseIdleConnections closes any connections which were previously 524 // connected from previous requests but are now sitting idle in 525 // a "keep-alive" state. It does not interrupt any connections currently 526 // in use. 527 func (t *Transport) CloseIdleConnections() { 528 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 529 t.idleMu.Lock() 530 m := t.idleConn 531 t.idleConn = nil 532 t.idleConnCh = nil 533 t.wantIdle = true 534 t.idleLRU = connLRU{} 535 t.idleMu.Unlock() 536 for _, conns := range m { 537 for _, pconn := range conns { 538 pconn.close(errCloseIdleConns) 539 } 540 } 541 if t2 := t.h2transport; t2 != nil { 542 t2.CloseIdleConnections() 543 } 544 } 545 546 var errCloseConn = errors.New("http: CloseConnection called") 547 548 // CloseConnections closes any connections which were previously 549 // connected from previous requests. It may interrupt connections 550 // currently in use. 551 func (t *Transport) CloseConnection(f func(raddr net.Addr) bool) { 552 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 553 t.idleMu.Lock() 554 m := t.idleConn 555 t.idleConn = nil 556 t.idleConnCh = nil 557 t.wantIdle = true 558 t.idleLRU = connLRU{} 559 t.idleMu.Unlock() 560 for _, conns := range m { 561 for _, pconn := range conns { 562 if f(pconn.conn.RemoteAddr()) { 563 pconn.close(errCloseConn) 564 } 565 } 566 } 567 if t2 := t.h2transport; t2 != nil { 568 if p, ok := t2.connPool().(*http2clientConnPool); ok { 569 p.mu.Lock() 570 defer p.mu.Unlock() 571 572 for _, vv := range p.conns { 573 for _, cc := range vv { 574 cc.mu.Lock() 575 if f(cc.tconn.RemoteAddr()) { 576 cc.closed = true 577 } 578 cc.mu.Unlock() 579 if cc.closed { 580 cc.tconn.Close() 581 } 582 } 583 } 584 } 585 } 586 } 587 588 // CancelRequest cancels an in-flight request by closing its connection. 589 // CancelRequest should only be called after RoundTrip has returned. 590 // 591 // Deprecated: Use Request.WithContext to create a request with a 592 // cancelable context instead. CancelRequest cannot cancel HTTP/2 593 // requests. 594 func (t *Transport) CancelRequest(req *Request) { 595 t.cancelRequest(req, errRequestCanceled) 596 } 597 598 // Cancel an in-flight request, recording the error value. 599 func (t *Transport) cancelRequest(req *Request, err error) { 600 t.reqMu.Lock() 601 cancel := t.reqCanceler[req] 602 delete(t.reqCanceler, req) 603 t.reqMu.Unlock() 604 if cancel != nil { 605 cancel(err) 606 } 607 } 608 609 // 610 // Private implementation past this point. 611 // 612 613 var ( 614 httpProxyEnv = &envOnce{ 615 names: []string{"HTTP_PROXY", "http_proxy"}, 616 } 617 httpsProxyEnv = &envOnce{ 618 names: []string{"HTTPS_PROXY", "https_proxy"}, 619 } 620 noProxyEnv = &envOnce{ 621 names: []string{"NO_PROXY", "no_proxy"}, 622 } 623 ) 624 625 // envOnce looks up an environment variable (optionally by multiple 626 // names) once. It mitigates expensive lookups on some platforms 627 // (e.g. Windows). 628 type envOnce struct { 629 names []string 630 once sync.Once 631 val string 632 } 633 634 func (e *envOnce) Get() string { 635 e.once.Do(e.init) 636 return e.val 637 } 638 639 func (e *envOnce) init() { 640 for _, n := range e.names { 641 e.val = os.Getenv(n) 642 if e.val != "" { 643 return 644 } 645 } 646 } 647 648 // reset is used by tests 649 func (e *envOnce) reset() { 650 e.once = sync.Once{} 651 e.val = "" 652 } 653 654 func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 655 if port := treq.URL.Port(); !validPort(port) { 656 return cm, fmt.Errorf("invalid URL port %q", port) 657 } 658 cm.GetConnectMethodAddr = t.GetConnectMethodAddr 659 cm.targetScheme = treq.URL.Scheme 660 cm.targetAddr = canonicalAddr(treq.URL) 661 if t.Proxy != nil { 662 cm.proxyURL, err = t.Proxy(treq.Request) 663 if err == nil && cm.proxyURL != nil { 664 if port := cm.proxyURL.Port(); !validPort(port) { 665 return cm, fmt.Errorf("invalid proxy URL port %q", port) 666 } 667 } 668 } 669 return cm, err 670 } 671 672 // proxyAuth returns the Proxy-Authorization header to set 673 // on requests, if applicable. 674 func (cm *connectMethod) proxyAuth() string { 675 if cm.proxyURL == nil { 676 return "" 677 } 678 if u := cm.proxyURL.User; u != nil { 679 username := u.Username() 680 password, _ := u.Password() 681 return "Basic " + basicAuth(username, password) 682 } 683 return "" 684 } 685 686 // error values for debugging and testing, not seen by users. 687 var ( 688 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") 689 errConnBroken = errors.New("http: putIdleConn: connection is in bad state") 690 errWantIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") 691 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") 692 errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") 693 errCloseIdleConns = errors.New("http: CloseIdleConnections called") 694 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") 695 errServerClosedIdle = errors.New("http: server closed idle connection") 696 errIdleConnTimeout = errors.New("http: idle connection timeout") 697 errNotCachingH2Conn = errors.New("http: not caching alternate protocol's connections") 698 ) 699 700 // transportReadFromServerError is used by Transport.readLoop when the 701 // 1 byte peek read fails and we're actually anticipating a response. 702 // Usually this is just due to the inherent keep-alive shut down race, 703 // where the server closed the connection at the same time the client 704 // wrote. The underlying err field is usually io.EOF or some 705 // ECONNRESET sort of thing which varies by platform. But it might be 706 // the user's custom net.Conn.Read error too, so we carry it along for 707 // them to return from Transport.RoundTrip. 708 type transportReadFromServerError struct { 709 err error 710 } 711 712 func (e transportReadFromServerError) Error() string { 713 return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) 714 } 715 716 func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { 717 if err := t.tryPutIdleConn(pconn); err != nil { 718 pconn.close(err) 719 } 720 } 721 722 func (t *Transport) maxIdleConnsPerHost() int { 723 if v := t.MaxIdleConnsPerHost; v != 0 { 724 return v 725 } 726 return DefaultMaxIdleConnsPerHost 727 } 728 729 // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting 730 // a new request. 731 // If pconn is no longer needed or not in a good state, tryPutIdleConn returns 732 // an error explaining why it wasn't registered. 733 // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. 734 func (t *Transport) tryPutIdleConn(pconn *persistConn) error { 735 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 736 return errKeepAlivesDisabled 737 } 738 if pconn.isBroken() { 739 return errConnBroken 740 } 741 if pconn.alt != nil { 742 return errNotCachingH2Conn 743 } 744 pconn.markReused() 745 key := pconn.cacheKey 746 747 t.idleMu.Lock() 748 defer t.idleMu.Unlock() 749 750 waitingDialer := t.idleConnCh[key] 751 select { 752 case waitingDialer <- pconn: 753 // We're done with this pconn and somebody else is 754 // currently waiting for a conn of this type (they're 755 // actively dialing, but this conn is ready 756 // first). Chrome calls this socket late binding. See 757 // https://insouciant.org/tech/connection-management-in-chromium/ 758 return nil 759 default: 760 if waitingDialer != nil { 761 // They had populated this, but their dial won 762 // first, so we can clean up this map entry. 763 delete(t.idleConnCh, key) 764 } 765 } 766 if t.wantIdle { 767 return errWantIdle 768 } 769 if t.idleConn == nil { 770 t.idleConn = make(map[connectMethodKey][]*persistConn) 771 } 772 idles := t.idleConn[key] 773 if len(idles) >= t.maxIdleConnsPerHost() { 774 return errTooManyIdleHost 775 } 776 for _, exist := range idles { 777 if exist == pconn { 778 log.Fatalf("dup idle pconn %p in freelist", pconn) 779 } 780 } 781 t.idleConn[key] = append(idles, pconn) 782 t.idleLRU.add(pconn) 783 if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { 784 oldest := t.idleLRU.removeOldest() 785 oldest.close(errTooManyIdle) 786 t.removeIdleConnLocked(oldest) 787 } 788 if t.IdleConnTimeout > 0 { 789 if pconn.idleTimer != nil { 790 pconn.idleTimer.Reset(t.IdleConnTimeout) 791 } else { 792 pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) 793 } 794 } 795 pconn.idleAt = time.Now() 796 return nil 797 } 798 799 // getIdleConnCh returns a channel to receive and return idle 800 // persistent connection for the given connectMethod. 801 // It may return nil, if persistent connections are not being used. 802 func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { 803 if t.DisableKeepAlives { 804 return nil 805 } 806 key := cm.key() 807 t.idleMu.Lock() 808 defer t.idleMu.Unlock() 809 t.wantIdle = false 810 if t.idleConnCh == nil { 811 t.idleConnCh = make(map[connectMethodKey]chan *persistConn) 812 } 813 ch, ok := t.idleConnCh[key] 814 if !ok { 815 ch = make(chan *persistConn) 816 t.idleConnCh[key] = ch 817 } 818 return ch 819 } 820 821 func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn, idleSince time.Time) { 822 key := cm.key() 823 t.idleMu.Lock() 824 defer t.idleMu.Unlock() 825 for { 826 pconns, ok := t.idleConn[key] 827 if !ok { 828 return nil, time.Time{} 829 } 830 if len(pconns) == 1 { 831 pconn = pconns[0] 832 delete(t.idleConn, key) 833 } else { 834 // 2 or more cached connections; use the most 835 // recently used one at the end. 836 pconn = pconns[len(pconns)-1] 837 t.idleConn[key] = pconns[:len(pconns)-1] 838 } 839 t.idleLRU.remove(pconn) 840 if pconn.isBroken() { 841 // There is a tiny window where this is 842 // possible, between the connecting dying and 843 // the persistConn readLoop calling 844 // Transport.removeIdleConn. Just skip it and 845 // carry on. 846 continue 847 } 848 if pconn.idleTimer != nil && !pconn.idleTimer.Stop() { 849 // We picked this conn at the ~same time it 850 // was expiring and it's trying to close 851 // itself in another goroutine. Don't use it. 852 continue 853 } 854 return pconn, pconn.idleAt 855 } 856 } 857 858 // removeIdleConn marks pconn as dead. 859 func (t *Transport) removeIdleConn(pconn *persistConn) { 860 t.idleMu.Lock() 861 defer t.idleMu.Unlock() 862 t.removeIdleConnLocked(pconn) 863 } 864 865 // t.idleMu must be held. 866 func (t *Transport) removeIdleConnLocked(pconn *persistConn) { 867 if pconn.idleTimer != nil { 868 pconn.idleTimer.Stop() 869 } 870 t.idleLRU.remove(pconn) 871 key := pconn.cacheKey 872 pconns := t.idleConn[key] 873 switch len(pconns) { 874 case 0: 875 // Nothing 876 case 1: 877 if pconns[0] == pconn { 878 delete(t.idleConn, key) 879 } 880 default: 881 for i, v := range pconns { 882 if v != pconn { 883 continue 884 } 885 // Slide down, keeping most recently-used 886 // conns at the end. 887 copy(pconns[i:], pconns[i+1:]) 888 t.idleConn[key] = pconns[:len(pconns)-1] 889 break 890 } 891 } 892 } 893 894 func (t *Transport) setReqCanceler(r *Request, fn func(error)) { 895 t.reqMu.Lock() 896 defer t.reqMu.Unlock() 897 if t.reqCanceler == nil { 898 t.reqCanceler = make(map[*Request]func(error)) 899 } 900 if fn != nil { 901 t.reqCanceler[r] = fn 902 } else { 903 delete(t.reqCanceler, r) 904 } 905 } 906 907 // replaceReqCanceler replaces an existing cancel function. If there is no cancel function 908 // for the request, we don't set the function and return false. 909 // Since CancelRequest will clear the canceler, we can use the return value to detect if 910 // the request was canceled since the last setReqCancel call. 911 func (t *Transport) replaceReqCanceler(r *Request, fn func(error)) bool { 912 t.reqMu.Lock() 913 defer t.reqMu.Unlock() 914 _, ok := t.reqCanceler[r] 915 if !ok { 916 return false 917 } 918 if fn != nil { 919 t.reqCanceler[r] = fn 920 } else { 921 delete(t.reqCanceler, r) 922 } 923 return true 924 } 925 926 var zeroDialer net.Dialer 927 928 func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { 929 if t.DialContext != nil { 930 return t.DialContext(ctx, network, addr) 931 } 932 if t.Dial != nil { 933 c, err := t.Dial(network, addr) 934 if c == nil && err == nil { 935 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") 936 } 937 return c, err 938 } 939 return zeroDialer.DialContext(ctx, network, addr) 940 } 941 942 // getConn dials and creates a new persistConn to the target as 943 // specified in the connectMethod. This includes doing a proxy CONNECT 944 // and/or setting up TLS. If this doesn't return an error, the persistConn 945 // is ready to write requests to. 946 func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) { 947 req := treq.Request 948 trace := treq.trace 949 ctx := req.Context() 950 if trace != nil && trace.GetConn != nil { 951 trace.GetConn(cm.addr()) 952 } 953 if pc, idleSince := t.getIdleConn(cm); pc != nil { 954 if trace != nil && trace.GotConn != nil { 955 trace.GotConn(pc.gotIdleConnTrace(idleSince)) 956 } 957 // set request canceler to some non-nil function so we 958 // can detect whether it was cleared between now and when 959 // we enter roundTrip 960 t.setReqCanceler(req, func(error) {}) 961 return pc, nil 962 } 963 964 type dialRes struct { 965 pc *persistConn 966 err error 967 } 968 dialc := make(chan dialRes) 969 970 // Copy these hooks so we don't race on the postPendingDial in 971 // the goroutine we launch. Issue 11136. 972 testHookPrePendingDial := testHookPrePendingDial 973 testHookPostPendingDial := testHookPostPendingDial 974 975 handlePendingDial := func() { 976 testHookPrePendingDial() 977 go func() { 978 if v := <-dialc; v.err == nil { 979 t.putOrCloseIdleConn(v.pc) 980 } 981 testHookPostPendingDial() 982 }() 983 } 984 985 cancelc := make(chan error, 1) 986 t.setReqCanceler(req, func(err error) { cancelc <- err }) 987 988 go func() { 989 pc, err := t.dialConn(ctx, cm) 990 dialc <- dialRes{pc, err} 991 }() 992 993 idleConnCh := t.getIdleConnCh(cm) 994 select { 995 case v := <-dialc: 996 // Our dial finished. 997 if v.pc != nil { 998 if trace != nil && trace.GotConn != nil && v.pc.alt == nil { 999 trace.GotConn(httptrace.GotConnInfo{Conn: v.pc.conn}) 1000 } 1001 return v.pc, nil 1002 } 1003 // Our dial failed. See why to return a nicer error 1004 // value. 1005 select { 1006 case <-req.Cancel: 1007 // It was an error due to cancelation, so prioritize that 1008 // error value. (Issue 16049) 1009 return nil, errRequestCanceledConn 1010 case <-req.Context().Done(): 1011 return nil, req.Context().Err() 1012 case err := <-cancelc: 1013 if err == errRequestCanceled { 1014 err = errRequestCanceledConn 1015 } 1016 return nil, err 1017 default: 1018 // It wasn't an error due to cancelation, so 1019 // return the original error message: 1020 return nil, v.err 1021 } 1022 case pc := <-idleConnCh: 1023 // Another request finished first and its net.Conn 1024 // became available before our dial. Or somebody 1025 // else's dial that they didn't use. 1026 // But our dial is still going, so give it away 1027 // when it finishes: 1028 handlePendingDial() 1029 if trace != nil && trace.GotConn != nil { 1030 trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()}) 1031 } 1032 return pc, nil 1033 case <-req.Cancel: 1034 handlePendingDial() 1035 return nil, errRequestCanceledConn 1036 case <-req.Context().Done(): 1037 handlePendingDial() 1038 return nil, req.Context().Err() 1039 case err := <-cancelc: 1040 handlePendingDial() 1041 if err == errRequestCanceled { 1042 err = errRequestCanceledConn 1043 } 1044 return nil, err 1045 } 1046 } 1047 1048 type oneConnDialer <-chan net.Conn 1049 1050 func newOneConnDialer(c net.Conn) proxy.Dialer { 1051 ch := make(chan net.Conn, 1) 1052 ch <- c 1053 return oneConnDialer(ch) 1054 } 1055 1056 func (d oneConnDialer) Dial(network, addr string) (net.Conn, error) { 1057 select { 1058 case c := <-d: 1059 return c, nil 1060 default: 1061 return nil, io.EOF 1062 } 1063 } 1064 1065 // The connect method and the transport can both specify a TLS 1066 // Host name. The transport's name takes precedence if present. 1067 func chooseTLSHost(cm connectMethod, t *Transport) string { 1068 tlsHost := "" 1069 if t.TLSClientConfig != nil { 1070 tlsHost = t.TLSClientConfig.ServerName 1071 } 1072 if tlsHost == "" { 1073 tlsHost = cm.tlsHost() 1074 } 1075 return tlsHost 1076 } 1077 1078 // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS 1079 // tunnel, this function establishes a nested TLS session inside the encrypted channel. 1080 // The remote endpoint's name may be overridden by TLSClientConfig.ServerName. 1081 func (pconn *persistConn) addTLS(name string, trace *httptrace.ClientTrace) error { 1082 // Initiate TLS and check remote host name against certificate. 1083 cfg := cloneTLSConfig(pconn.t.TLSClientConfig) 1084 if cfg.ServerName == "" { 1085 cfg.ServerName = name 1086 } 1087 plainConn := pconn.conn 1088 tlsConn := tls.Client(plainConn, cfg) 1089 errc := make(chan error, 2) 1090 var timer *time.Timer // for canceling TLS handshake 1091 if d := pconn.t.TLSHandshakeTimeout; d != 0 { 1092 timer = time.AfterFunc(d, func() { 1093 errc <- tlsHandshakeTimeoutError{} 1094 }) 1095 } 1096 go func() { 1097 if trace != nil && trace.TLSHandshakeStart != nil { 1098 trace.TLSHandshakeStart() 1099 } 1100 err := tlsConn.Handshake() 1101 if timer != nil { 1102 timer.Stop() 1103 } 1104 errc <- err 1105 }() 1106 if err := <-errc; err != nil { 1107 plainConn.Close() 1108 if trace != nil && trace.TLSHandshakeDone != nil { 1109 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1110 } 1111 return err 1112 } 1113 if !cfg.InsecureSkipVerify { 1114 if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { 1115 plainConn.Close() 1116 return err 1117 } 1118 } 1119 cs := tlsConn.ConnectionState() 1120 if trace != nil && trace.TLSHandshakeDone != nil { 1121 trace.TLSHandshakeDone(cs, nil) 1122 } 1123 pconn.tlsState = &cs 1124 pconn.conn = tlsConn 1125 return nil 1126 } 1127 1128 func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) { 1129 pconn := &persistConn{ 1130 t: t, 1131 cacheKey: cm.key(), 1132 reqch: make(chan requestAndChan, 1), 1133 writech: make(chan writeRequest, 1), 1134 closech: make(chan struct{}), 1135 writeErrCh: make(chan error, 1), 1136 writeLoopDone: make(chan struct{}), 1137 } 1138 trace := httptrace.ContextClientTrace(ctx) 1139 wrapErr := func(err error) error { 1140 if cm.proxyURL != nil { 1141 // Return a typed error, per Issue 16997 1142 return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} 1143 } 1144 return err 1145 } 1146 if cm.scheme() == "https" && t.DialTLS != nil { 1147 var err error 1148 pconn.conn, err = t.DialTLS("tcp", cm.addr()) 1149 if err != nil { 1150 return nil, wrapErr(err) 1151 } 1152 if pconn.conn == nil { 1153 return nil, wrapErr(errors.New("net/http: Transport.DialTLS returned (nil, nil)")) 1154 } 1155 if tc, ok := pconn.conn.(*tls.Conn); ok { 1156 // Handshake here, in case DialTLS didn't. TLSNextProto below 1157 // depends on it for knowing the connection state. 1158 if trace != nil && trace.TLSHandshakeStart != nil { 1159 trace.TLSHandshakeStart() 1160 } 1161 if err := tc.Handshake(); err != nil { 1162 go pconn.conn.Close() 1163 if trace != nil && trace.TLSHandshakeDone != nil { 1164 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1165 } 1166 return nil, err 1167 } 1168 cs := tc.ConnectionState() 1169 if trace != nil && trace.TLSHandshakeDone != nil { 1170 trace.TLSHandshakeDone(cs, nil) 1171 } 1172 pconn.tlsState = &cs 1173 } 1174 } else { 1175 conn, err := t.dial(ctx, "tcp", cm.addr()) 1176 if err != nil { 1177 return nil, wrapErr(err) 1178 } 1179 pconn.conn = conn 1180 if cm.scheme() == "https" { 1181 var firstTLSHost string 1182 if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { 1183 return nil, wrapErr(err) 1184 } 1185 if err = pconn.addTLS(firstTLSHost, trace); err != nil { 1186 return nil, wrapErr(err) 1187 } 1188 } 1189 } 1190 1191 // Proxy setup. 1192 switch { 1193 case cm.proxyURL == nil: 1194 // Do nothing. Not using a proxy. 1195 case cm.proxyURL.Scheme == "socks5": 1196 conn := pconn.conn 1197 var auth *proxy.Auth 1198 if u := cm.proxyURL.User; u != nil { 1199 auth = &proxy.Auth{} 1200 auth.User = u.Username() 1201 auth.Password, _ = u.Password() 1202 } 1203 p, err := proxy.SOCKS5("", cm.addr(), auth, newOneConnDialer(conn)) 1204 if err != nil { 1205 conn.Close() 1206 return nil, err 1207 } 1208 if _, err := p.Dial("tcp", cm.targetAddr); err != nil { 1209 conn.Close() 1210 return nil, err 1211 } 1212 case cm.targetScheme == "http": 1213 pconn.isProxy = true 1214 if pa := cm.proxyAuth(); pa != "" { 1215 pconn.mutateHeaderFunc = func(h Header) { 1216 h.Set("Proxy-Authorization", pa) 1217 } 1218 } 1219 case cm.targetScheme == "https": 1220 conn := pconn.conn 1221 hdr := t.ProxyConnectHeader 1222 if hdr == nil { 1223 hdr = make(Header) 1224 } 1225 connectReq := &Request{ 1226 Method: "CONNECT", 1227 URL: &url.URL{Opaque: cm.targetAddr}, 1228 Host: cm.targetAddr, 1229 Header: hdr, 1230 } 1231 if pa := cm.proxyAuth(); pa != "" { 1232 connectReq.Header.Set("Proxy-Authorization", pa) 1233 } 1234 connectReq.Write(conn) 1235 1236 // Read response. 1237 // Okay to use and discard buffered reader here, because 1238 // TLS server will not speak until spoken to. 1239 br := bufio.NewReader(conn) 1240 resp, err := ReadResponse(br, connectReq) 1241 if err != nil { 1242 conn.Close() 1243 return nil, err 1244 } 1245 if resp.StatusCode != 200 { 1246 f := strings.SplitN(resp.Status, " ", 2) 1247 conn.Close() 1248 if len(f) < 2 { 1249 return nil, errors.New("unknown status code") 1250 } 1251 return nil, errors.New(f[1]) 1252 } 1253 } 1254 1255 if cm.proxyURL != nil && cm.targetScheme == "https" { 1256 if err := pconn.addTLS(cm.tlsHost(), trace); err != nil { 1257 return nil, err 1258 } 1259 } 1260 1261 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 1262 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 1263 return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil 1264 } 1265 } 1266 1267 pconn.br = bufio.NewReader(pconn) 1268 pconn.bw = bufio.NewWriter(persistConnWriter{pconn}) 1269 go pconn.readLoop() 1270 go pconn.writeLoop() 1271 return pconn, nil 1272 } 1273 1274 // persistConnWriter is the io.Writer written to by pc.bw. 1275 // It accumulates the number of bytes written to the underlying conn, 1276 // so the retry logic can determine whether any bytes made it across 1277 // the wire. 1278 // This is exactly 1 pointer field wide so it can go into an interface 1279 // without allocation. 1280 type persistConnWriter struct { 1281 pc *persistConn 1282 } 1283 1284 func (w persistConnWriter) Write(p []byte) (n int, err error) { 1285 n, err = w.pc.conn.Write(p) 1286 w.pc.nwrite += int64(n) 1287 return 1288 } 1289 1290 // useProxy reports whether requests to addr should use a proxy, 1291 // according to the NO_PROXY or no_proxy environment variable. 1292 // addr is always a canonicalAddr with a host and port. 1293 func useProxy(addr string) bool { 1294 if len(addr) == 0 { 1295 return true 1296 } 1297 host, _, err := net.SplitHostPort(addr) 1298 if err != nil { 1299 return false 1300 } 1301 if host == "localhost" { 1302 return false 1303 } 1304 if ip := net.ParseIP(host); ip != nil { 1305 if ip.IsLoopback() { 1306 return false 1307 } 1308 } 1309 1310 noProxy := noProxyEnv.Get() 1311 if noProxy == "*" { 1312 return false 1313 } 1314 1315 addr = strings.ToLower(strings.TrimSpace(addr)) 1316 if hasPort(addr) { 1317 addr = addr[:strings.LastIndex(addr, ":")] 1318 } 1319 1320 for _, p := range strings.Split(noProxy, ",") { 1321 p = strings.ToLower(strings.TrimSpace(p)) 1322 if len(p) == 0 { 1323 continue 1324 } 1325 if hasPort(p) { 1326 p = p[:strings.LastIndex(p, ":")] 1327 } 1328 if addr == p { 1329 return false 1330 } 1331 if len(p) == 0 { 1332 // There is no host part, likely the entry is malformed; ignore. 1333 continue 1334 } 1335 if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { 1336 // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" 1337 return false 1338 } 1339 if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { 1340 // no_proxy "foo.com" matches "bar.foo.com" 1341 return false 1342 } 1343 } 1344 return true 1345 } 1346 1347 // connectMethod is the map key (in its String form) for keeping persistent 1348 // TCP connections alive for subsequent HTTP requests. 1349 // 1350 // A connect method may be of the following types: 1351 // 1352 // Cache key form Description 1353 // ----------------- ------------------------- 1354 // |http|foo.com http directly to server, no proxy 1355 // |https|foo.com https directly to server, no proxy 1356 // http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 1357 // http://proxy.com|http http to proxy, http to anywhere after that 1358 // socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com 1359 // socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com 1360 // https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com 1361 // https://proxy.com|http https to proxy, http to anywhere after that 1362 // 1363 type connectMethod struct { 1364 proxyURL *url.URL // nil for no proxy, else full proxy URL 1365 targetScheme string // "http" or "https" 1366 // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), 1367 // then targetAddr is not included in the connect method key, because the socket can 1368 // be reused for different targetAddr values. 1369 targetAddr string 1370 1371 GetConnectMethodAddr func(addr string) string 1372 } 1373 1374 func (cm *connectMethod) key() connectMethodKey { 1375 proxyStr := "" 1376 targetAddr := cm.targetAddr 1377 if cm.GetConnectMethodAddr != nil { 1378 targetAddr = cm.GetConnectMethodAddr(targetAddr) 1379 } 1380 if cm.proxyURL != nil { 1381 proxyStr = cm.proxyURL.String() 1382 if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { 1383 targetAddr = "" 1384 } 1385 } 1386 return connectMethodKey{ 1387 proxy: proxyStr, 1388 scheme: cm.targetScheme, 1389 addr: targetAddr, 1390 } 1391 } 1392 1393 // scheme returns the first hop scheme: http, https, or socks5 1394 func (cm *connectMethod) scheme() string { 1395 if cm.proxyURL != nil { 1396 return cm.proxyURL.Scheme 1397 } 1398 return cm.targetScheme 1399 } 1400 1401 // addr returns the first hop "host:port" to which we need to TCP connect. 1402 func (cm *connectMethod) addr() string { 1403 if cm.proxyURL != nil { 1404 return canonicalAddr(cm.proxyURL) 1405 } 1406 return cm.targetAddr 1407 } 1408 1409 // tlsHost returns the host name to match against the peer's 1410 // TLS certificate. 1411 func (cm *connectMethod) tlsHost() string { 1412 h := cm.targetAddr 1413 if hasPort(h) { 1414 h = h[:strings.LastIndex(h, ":")] 1415 } 1416 return h 1417 } 1418 1419 // connectMethodKey is the map key version of connectMethod, with a 1420 // stringified proxy URL (or the empty string) instead of a pointer to 1421 // a URL. 1422 type connectMethodKey struct { 1423 proxy, scheme, addr string 1424 } 1425 1426 func (k connectMethodKey) String() string { 1427 // Only used by tests. 1428 return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr) 1429 } 1430 1431 // persistConn wraps a connection, usually a persistent one 1432 // (but may be used for non-keep-alive requests as well) 1433 type persistConn struct { 1434 // alt optionally specifies the TLS NextProto RoundTripper. 1435 // This is used for HTTP/2 today and future protocols later. 1436 // If it's non-nil, the rest of the fields are unused. 1437 alt RoundTripper 1438 1439 t *Transport 1440 cacheKey connectMethodKey 1441 conn net.Conn 1442 tlsState *tls.ConnectionState 1443 br *bufio.Reader // from conn 1444 bw *bufio.Writer // to conn 1445 nwrite int64 // bytes written 1446 reqch chan requestAndChan // written by roundTrip; read by readLoop 1447 writech chan writeRequest // written by roundTrip; read by writeLoop 1448 closech chan struct{} // closed when conn closed 1449 isProxy bool 1450 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 1451 readLimit int64 // bytes allowed to be read; owned by readLoop 1452 // writeErrCh passes the request write error (usually nil) 1453 // from the writeLoop goroutine to the readLoop which passes 1454 // it off to the res.Body reader, which then uses it to decide 1455 // whether or not a connection can be reused. Issue 7569. 1456 writeErrCh chan error 1457 1458 writeLoopDone chan struct{} // closed when write loop ends 1459 1460 // Both guarded by Transport.idleMu: 1461 idleAt time.Time // time it last become idle 1462 idleTimer *time.Timer // holding an AfterFunc to close it 1463 1464 mu sync.Mutex // guards following fields 1465 numExpectedResponses int 1466 closed error // set non-nil when conn is closed, before closech is closed 1467 canceledErr error // set non-nil if conn is canceled 1468 broken bool // an error has happened on this connection; marked broken so it's not reused. 1469 reused bool // whether conn has had successful request/response and is being reused. 1470 // mutateHeaderFunc is an optional func to modify extra 1471 // headers on each outbound request before it's written. (the 1472 // original Request given to RoundTrip is not modified) 1473 mutateHeaderFunc func(Header) 1474 } 1475 1476 func (pc *persistConn) maxHeaderResponseSize() int64 { 1477 if v := pc.t.MaxResponseHeaderBytes; v != 0 { 1478 return v 1479 } 1480 return 10 << 20 // conservative default; same as http2 1481 } 1482 1483 func (pc *persistConn) Read(p []byte) (n int, err error) { 1484 if pc.readLimit <= 0 { 1485 return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) 1486 } 1487 if int64(len(p)) > pc.readLimit { 1488 p = p[:pc.readLimit] 1489 } 1490 n, err = pc.conn.Read(p) 1491 if err == io.EOF { 1492 pc.sawEOF = true 1493 } 1494 pc.readLimit -= int64(n) 1495 return 1496 } 1497 1498 // isBroken reports whether this connection is in a known broken state. 1499 func (pc *persistConn) isBroken() bool { 1500 pc.mu.Lock() 1501 b := pc.closed != nil 1502 pc.mu.Unlock() 1503 return b 1504 } 1505 1506 // canceled returns non-nil if the connection was closed due to 1507 // CancelRequest or due to context cancelation. 1508 func (pc *persistConn) canceled() error { 1509 pc.mu.Lock() 1510 defer pc.mu.Unlock() 1511 return pc.canceledErr 1512 } 1513 1514 // isReused reports whether this connection is in a known broken state. 1515 func (pc *persistConn) isReused() bool { 1516 pc.mu.Lock() 1517 r := pc.reused 1518 pc.mu.Unlock() 1519 return r 1520 } 1521 1522 func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { 1523 pc.mu.Lock() 1524 defer pc.mu.Unlock() 1525 t.Reused = pc.reused 1526 t.Conn = pc.conn 1527 t.WasIdle = true 1528 if !idleAt.IsZero() { 1529 t.IdleTime = time.Since(idleAt) 1530 } 1531 return 1532 } 1533 1534 func (pc *persistConn) cancelRequest(err error) { 1535 pc.mu.Lock() 1536 defer pc.mu.Unlock() 1537 pc.canceledErr = err 1538 pc.closeLocked(errRequestCanceled) 1539 } 1540 1541 // closeConnIfStillIdle closes the connection if it's still sitting idle. 1542 // This is what's called by the persistConn's idleTimer, and is run in its 1543 // own goroutine. 1544 func (pc *persistConn) closeConnIfStillIdle() { 1545 t := pc.t 1546 t.idleMu.Lock() 1547 defer t.idleMu.Unlock() 1548 if _, ok := t.idleLRU.m[pc]; !ok { 1549 // Not idle. 1550 return 1551 } 1552 t.removeIdleConnLocked(pc) 1553 pc.close(errIdleConnTimeout) 1554 } 1555 1556 // mapRoundTripError returns the appropriate error value for 1557 // persistConn.roundTrip. 1558 // 1559 // The provided err is the first error that (*persistConn).roundTrip 1560 // happened to receive from its select statement. 1561 // 1562 // The startBytesWritten value should be the value of pc.nwrite before the roundTrip 1563 // started writing the request. 1564 func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { 1565 if err == nil { 1566 return nil 1567 } 1568 1569 // If the request was canceled, that's better than network 1570 // failures that were likely the result of tearing down the 1571 // connection. 1572 if cerr := pc.canceled(); cerr != nil { 1573 return cerr 1574 } 1575 1576 // See if an error was set explicitly. 1577 req.mu.Lock() 1578 reqErr := req.err 1579 req.mu.Unlock() 1580 if reqErr != nil { 1581 return reqErr 1582 } 1583 1584 if err == errServerClosedIdle { 1585 // Don't decorate 1586 return err 1587 } 1588 1589 if _, ok := err.(transportReadFromServerError); ok { 1590 // Don't decorate 1591 return err 1592 } 1593 if pc.isBroken() { 1594 <-pc.writeLoopDone 1595 if pc.nwrite == startBytesWritten { 1596 return nothingWrittenError{err} 1597 } 1598 return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) 1599 } 1600 return err 1601 } 1602 1603 func (pc *persistConn) readLoop() { 1604 closeErr := errReadLoopExiting // default value, if not changed below 1605 defer func() { 1606 pc.close(closeErr) 1607 pc.t.removeIdleConn(pc) 1608 }() 1609 1610 tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { 1611 if err := pc.t.tryPutIdleConn(pc); err != nil { 1612 closeErr = err 1613 if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { 1614 trace.PutIdleConn(err) 1615 } 1616 return false 1617 } 1618 if trace != nil && trace.PutIdleConn != nil { 1619 trace.PutIdleConn(nil) 1620 } 1621 return true 1622 } 1623 1624 // eofc is used to block caller goroutines reading from Response.Body 1625 // at EOF until this goroutines has (potentially) added the connection 1626 // back to the idle pool. 1627 eofc := make(chan struct{}) 1628 defer close(eofc) // unblock reader on errors 1629 1630 // Read this once, before loop starts. (to avoid races in tests) 1631 testHookMu.Lock() 1632 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 1633 testHookMu.Unlock() 1634 1635 alive := true 1636 for alive { 1637 pc.readLimit = pc.maxHeaderResponseSize() 1638 _, err := pc.br.Peek(1) 1639 1640 pc.mu.Lock() 1641 if pc.numExpectedResponses == 0 { 1642 pc.readLoopPeekFailLocked(err) 1643 pc.mu.Unlock() 1644 return 1645 } 1646 pc.mu.Unlock() 1647 1648 rc := <-pc.reqch 1649 trace := httptrace.ContextClientTrace(rc.req.Context()) 1650 1651 var resp *Response 1652 if err == nil { 1653 resp, err = pc.readResponse(rc, trace) 1654 } else { 1655 err = transportReadFromServerError{err} 1656 closeErr = err 1657 } 1658 1659 if err != nil { 1660 if pc.readLimit <= 0 { 1661 err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) 1662 } 1663 1664 select { 1665 case rc.ch <- responseAndError{err: err}: 1666 case <-rc.callerGone: 1667 return 1668 } 1669 return 1670 } 1671 pc.readLimit = maxInt64 // effictively no limit for response bodies 1672 1673 pc.mu.Lock() 1674 pc.numExpectedResponses-- 1675 pc.mu.Unlock() 1676 1677 hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 1678 1679 if resp.Close || rc.req.Close || resp.StatusCode <= 199 { 1680 // Don't do keep-alive on error if either party requested a close 1681 // or we get an unexpected informational (1xx) response. 1682 // StatusCode 100 is already handled above. 1683 alive = false 1684 } 1685 1686 if !hasBody { 1687 pc.t.setReqCanceler(rc.req, nil) 1688 1689 // Put the idle conn back into the pool before we send the response 1690 // so if they process it quickly and make another request, they'll 1691 // get this same conn. But we use the unbuffered channel 'rc' 1692 // to guarantee that persistConn.roundTrip got out of its select 1693 // potentially waiting for this persistConn to close. 1694 // but after 1695 alive = alive && 1696 !pc.sawEOF && 1697 pc.wroteRequest() && 1698 tryPutIdleConn(trace) 1699 1700 select { 1701 case rc.ch <- responseAndError{res: resp}: 1702 case <-rc.callerGone: 1703 return 1704 } 1705 1706 // Now that they've read from the unbuffered channel, they're safely 1707 // out of the select that also waits on this goroutine to die, so 1708 // we're allowed to exit now if needed (if alive is false) 1709 testHookReadLoopBeforeNextRead() 1710 continue 1711 } 1712 1713 waitForBodyRead := make(chan bool, 2) 1714 body := &bodyEOFSignal{ 1715 body: resp.Body, 1716 earlyCloseFn: func() error { 1717 waitForBodyRead <- false 1718 <-eofc // will be closed by deferred call at the end of the function 1719 return nil 1720 1721 }, 1722 fn: func(err error) error { 1723 isEOF := err == io.EOF 1724 waitForBodyRead <- isEOF 1725 if isEOF { 1726 <-eofc // see comment above eofc declaration 1727 } else if err != nil { 1728 if cerr := pc.canceled(); cerr != nil { 1729 return cerr 1730 } 1731 } 1732 return err 1733 }, 1734 } 1735 1736 resp.Body = body 1737 if rc.addedGzip && strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { 1738 resp.Body = &gzipReader{body: body} 1739 resp.Header.Del("Content-Encoding") 1740 resp.Header.Del("Content-Length") 1741 resp.ContentLength = -1 1742 resp.Uncompressed = true 1743 } 1744 1745 select { 1746 case rc.ch <- responseAndError{res: resp}: 1747 case <-rc.callerGone: 1748 return 1749 } 1750 1751 // Before looping back to the top of this function and peeking on 1752 // the bufio.Reader, wait for the caller goroutine to finish 1753 // reading the response body. (or for cancelation or death) 1754 select { 1755 case bodyEOF := <-waitForBodyRead: 1756 pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool 1757 alive = alive && 1758 bodyEOF && 1759 !pc.sawEOF && 1760 pc.wroteRequest() && 1761 tryPutIdleConn(trace) 1762 if bodyEOF { 1763 eofc <- struct{}{} 1764 } 1765 case <-rc.req.Cancel: 1766 alive = false 1767 pc.t.CancelRequest(rc.req) 1768 case <-rc.req.Context().Done(): 1769 alive = false 1770 pc.t.cancelRequest(rc.req, rc.req.Context().Err()) 1771 case <-pc.closech: 1772 alive = false 1773 } 1774 1775 testHookReadLoopBeforeNextRead() 1776 } 1777 } 1778 1779 func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 1780 if pc.closed != nil { 1781 return 1782 } 1783 if n := pc.br.Buffered(); n > 0 { 1784 buf, _ := pc.br.Peek(n) 1785 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 1786 } 1787 if peekErr == io.EOF { 1788 // common case. 1789 pc.closeLocked(errServerClosedIdle) 1790 } else { 1791 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr)) 1792 } 1793 } 1794 1795 // readResponse reads an HTTP response (or two, in the case of "Expect: 1796 // 100-continue") from the server. It returns the final non-100 one. 1797 // trace is optional. 1798 func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { 1799 if trace != nil && trace.GotFirstResponseByte != nil { 1800 if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { 1801 trace.GotFirstResponseByte() 1802 } 1803 } 1804 resp, err = ReadResponse(pc.br, rc.req) 1805 if err != nil { 1806 return 1807 } 1808 if rc.continueCh != nil { 1809 if resp.StatusCode == 100 { 1810 if trace != nil && trace.Got100Continue != nil { 1811 trace.Got100Continue() 1812 } 1813 rc.continueCh <- struct{}{} 1814 } else { 1815 close(rc.continueCh) 1816 } 1817 } 1818 if resp.StatusCode == 100 { 1819 pc.readLimit = pc.maxHeaderResponseSize() // reset the limit 1820 resp, err = ReadResponse(pc.br, rc.req) 1821 if err != nil { 1822 return 1823 } 1824 } 1825 resp.TLS = pc.tlsState 1826 return 1827 } 1828 1829 // waitForContinue returns the function to block until 1830 // any response, timeout or connection close. After any of them, 1831 // the function returns a bool which indicates if the body should be sent. 1832 func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 1833 if continueCh == nil { 1834 return nil 1835 } 1836 return func() bool { 1837 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 1838 defer timer.Stop() 1839 1840 select { 1841 case _, ok := <-continueCh: 1842 return ok 1843 case <-timer.C: 1844 return true 1845 case <-pc.closech: 1846 return false 1847 } 1848 } 1849 } 1850 1851 // nothingWrittenError wraps a write errors which ended up writing zero bytes. 1852 type nothingWrittenError struct { 1853 error 1854 } 1855 1856 func (pc *persistConn) writeLoop() { 1857 defer close(pc.writeLoopDone) 1858 for { 1859 select { 1860 case wr := <-pc.writech: 1861 startBytesWritten := pc.nwrite 1862 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 1863 if bre, ok := err.(requestBodyReadError); ok { 1864 err = bre.error 1865 // Errors reading from the user's 1866 // Request.Body are high priority. 1867 // Set it here before sending on the 1868 // channels below or calling 1869 // pc.close() which tears town 1870 // connections and causes other 1871 // errors. 1872 wr.req.setError(err) 1873 } 1874 if err == nil { 1875 err = pc.bw.Flush() 1876 } 1877 if err != nil { 1878 wr.req.Request.closeBody() 1879 if pc.nwrite == startBytesWritten { 1880 err = nothingWrittenError{err} 1881 } 1882 } 1883 pc.writeErrCh <- err // to the body reader, which might recycle us 1884 wr.ch <- err // to the roundTrip function 1885 if err != nil { 1886 pc.close(err) 1887 return 1888 } 1889 case <-pc.closech: 1890 return 1891 } 1892 } 1893 } 1894 1895 // wroteRequest is a check before recycling a connection that the previous write 1896 // (from writeLoop above) happened and was successful. 1897 func (pc *persistConn) wroteRequest() bool { 1898 select { 1899 case err := <-pc.writeErrCh: 1900 // Common case: the write happened well before the response, so 1901 // avoid creating a timer. 1902 return err == nil 1903 default: 1904 // Rare case: the request was written in writeLoop above but 1905 // before it could send to pc.writeErrCh, the reader read it 1906 // all, processed it, and called us here. In this case, give the 1907 // write goroutine a bit of time to finish its send. 1908 // 1909 // Less rare case: We also get here in the legitimate case of 1910 // Issue 7569, where the writer is still writing (or stalled), 1911 // but the server has already replied. In this case, we don't 1912 // want to wait too long, and we want to return false so this 1913 // connection isn't re-used. 1914 select { 1915 case err := <-pc.writeErrCh: 1916 return err == nil 1917 case <-time.After(50 * time.Millisecond): 1918 return false 1919 } 1920 } 1921 } 1922 1923 // responseAndError is how the goroutine reading from an HTTP/1 server 1924 // communicates with the goroutine doing the RoundTrip. 1925 type responseAndError struct { 1926 res *Response // else use this response (see res method) 1927 err error 1928 } 1929 1930 type requestAndChan struct { 1931 req *Request 1932 ch chan responseAndError // unbuffered; always send in select on callerGone 1933 1934 // whether the Transport (as opposed to the user client code) 1935 // added the Accept-Encoding gzip header. If the Transport 1936 // set it, only then do we transparently decode the gzip. 1937 addedGzip bool 1938 1939 // Optional blocking chan for Expect: 100-continue (for send). 1940 // If the request has an "Expect: 100-continue" header and 1941 // the server responds 100 Continue, readLoop send a value 1942 // to writeLoop via this chan. 1943 continueCh chan<- struct{} 1944 1945 callerGone <-chan struct{} // closed when roundTrip caller has returned 1946 } 1947 1948 // A writeRequest is sent by the readLoop's goroutine to the 1949 // writeLoop's goroutine to write a request while the read loop 1950 // concurrently waits on both the write response and the server's 1951 // reply. 1952 type writeRequest struct { 1953 req *transportRequest 1954 ch chan<- error 1955 1956 // Optional blocking chan for Expect: 100-continue (for receive). 1957 // If not nil, writeLoop blocks sending request body until 1958 // it receives from this chan. 1959 continueCh <-chan struct{} 1960 } 1961 1962 type httpError struct { 1963 err string 1964 timeout bool 1965 } 1966 1967 func (e *httpError) Error() string { return e.err } 1968 func (e *httpError) Timeout() bool { return e.timeout } 1969 func (e *httpError) Temporary() bool { return true } 1970 1971 var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} 1972 var errRequestCanceled = errors.New("net/http: request canceled") 1973 var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? 1974 1975 func nop() {} 1976 1977 // testHooks. Always non-nil. 1978 var ( 1979 testHookEnterRoundTrip = nop 1980 testHookWaitResLoop = nop 1981 testHookRoundTripRetried = nop 1982 testHookPrePendingDial = nop 1983 testHookPostPendingDial = nop 1984 1985 testHookMu sync.Locker = fakeLocker{} // guards following 1986 testHookReadLoopBeforeNextRead = nop 1987 ) 1988 1989 func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 1990 testHookEnterRoundTrip() 1991 if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { 1992 pc.t.putOrCloseIdleConn(pc) 1993 return nil, errRequestCanceled 1994 } 1995 pc.mu.Lock() 1996 pc.numExpectedResponses++ 1997 headerFn := pc.mutateHeaderFunc 1998 pc.mu.Unlock() 1999 2000 if headerFn != nil { 2001 headerFn(req.extraHeaders()) 2002 } 2003 2004 // Ask for a compressed version if the caller didn't set their 2005 // own value for Accept-Encoding. We only attempt to 2006 // uncompress the gzip stream if we were the layer that 2007 // requested it. 2008 requestedGzip := false 2009 if !pc.t.DisableCompression && 2010 req.Header.Get("Accept-Encoding") == "" && 2011 req.Header.Get("Range") == "" && 2012 req.Method != "HEAD" { 2013 // Request gzip only, not deflate. Deflate is ambiguous and 2014 // not as universally supported anyway. 2015 // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 2016 // 2017 // Note that we don't request this for HEAD requests, 2018 // due to a bug in nginx: 2019 // http://trac.nginx.org/nginx/ticket/358 2020 // https://golang.org/issue/5522 2021 // 2022 // We don't request gzip if the request is for a range, since 2023 // auto-decoding a portion of a gzipped document will just fail 2024 // anyway. See https://golang.org/issue/8923 2025 requestedGzip = true 2026 req.extraHeaders().Set("Accept-Encoding", "gzip") 2027 } 2028 2029 var continueCh chan struct{} 2030 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 2031 continueCh = make(chan struct{}, 1) 2032 } 2033 2034 if pc.t.DisableKeepAlives { 2035 req.extraHeaders().Set("Connection", "close") 2036 } 2037 2038 gone := make(chan struct{}) 2039 defer close(gone) 2040 2041 defer func() { 2042 if err != nil { 2043 pc.t.setReqCanceler(req.Request, nil) 2044 } 2045 }() 2046 2047 const debugRoundTrip = false 2048 2049 // Write the request concurrently with waiting for a response, 2050 // in case the server decides to reply before reading our full 2051 // request body. 2052 startBytesWritten := pc.nwrite 2053 writeErrCh := make(chan error, 1) 2054 pc.writech <- writeRequest{req, writeErrCh, continueCh} 2055 2056 resc := make(chan responseAndError) 2057 pc.reqch <- requestAndChan{ 2058 req: req.Request, 2059 ch: resc, 2060 addedGzip: requestedGzip, 2061 continueCh: continueCh, 2062 callerGone: gone, 2063 } 2064 2065 var respHeaderTimer <-chan time.Time 2066 cancelChan := req.Request.Cancel 2067 ctxDoneChan := req.Context().Done() 2068 for { 2069 testHookWaitResLoop() 2070 select { 2071 case err := <-writeErrCh: 2072 if debugRoundTrip { 2073 req.logf("writeErrCh resv: %T/%#v", err, err) 2074 } 2075 if err != nil { 2076 pc.close(fmt.Errorf("write error: %v", err)) 2077 return nil, pc.mapRoundTripError(req, startBytesWritten, err) 2078 } 2079 if d := pc.t.ResponseHeaderTimeout; d > 0 { 2080 if debugRoundTrip { 2081 req.logf("starting timer for %v", d) 2082 } 2083 timer := time.NewTimer(d) 2084 defer timer.Stop() // prevent leaks 2085 respHeaderTimer = timer.C 2086 } 2087 case <-pc.closech: 2088 if debugRoundTrip { 2089 req.logf("closech recv: %T %#v", pc.closed, pc.closed) 2090 } 2091 return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) 2092 case <-respHeaderTimer: 2093 if debugRoundTrip { 2094 req.logf("timeout waiting for response headers.") 2095 } 2096 pc.close(errTimeout) 2097 return nil, errTimeout 2098 case re := <-resc: 2099 if (re.res == nil) == (re.err == nil) { 2100 panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) 2101 } 2102 if debugRoundTrip { 2103 req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) 2104 } 2105 if re.err != nil { 2106 return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) 2107 } 2108 return re.res, nil 2109 case <-cancelChan: 2110 pc.t.CancelRequest(req.Request) 2111 cancelChan = nil 2112 case <-ctxDoneChan: 2113 pc.t.cancelRequest(req.Request, req.Context().Err()) 2114 cancelChan = nil 2115 ctxDoneChan = nil 2116 } 2117 } 2118 } 2119 2120 // tLogKey is a context WithValue key for test debugging contexts containing 2121 // a t.Logf func. See export_test.go's Request.WithT method. 2122 type tLogKey struct{} 2123 2124 func (tr *transportRequest) logf(format string, args ...interface{}) { 2125 if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...interface{})); ok { 2126 logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) 2127 } 2128 } 2129 2130 // markReused marks this connection as having been successfully used for a 2131 // request and response. 2132 func (pc *persistConn) markReused() { 2133 pc.mu.Lock() 2134 pc.reused = true 2135 pc.mu.Unlock() 2136 } 2137 2138 // close closes the underlying TCP connection and closes 2139 // the pc.closech channel. 2140 // 2141 // The provided err is only for testing and debugging; in normal 2142 // circumstances it should never be seen by users. 2143 func (pc *persistConn) close(err error) { 2144 pc.mu.Lock() 2145 defer pc.mu.Unlock() 2146 pc.closeLocked(err) 2147 } 2148 2149 func (pc *persistConn) closeLocked(err error) { 2150 if err == nil { 2151 panic("nil error") 2152 } 2153 pc.broken = true 2154 if pc.closed == nil { 2155 pc.closed = err 2156 if pc.alt != nil { 2157 // Do nothing; can only get here via getConn's 2158 // handlePendingDial's putOrCloseIdleConn when 2159 // it turns out the abandoned connection in 2160 // flight ended up negotiating an alternate 2161 // protocol. We don't use the connection 2162 // freelist for http2. That's done by the 2163 // alternate protocol's RoundTripper. 2164 } else { 2165 pc.conn.Close() 2166 close(pc.closech) 2167 } 2168 } 2169 pc.mutateHeaderFunc = nil 2170 } 2171 2172 var portMap = map[string]string{ 2173 "http": "80", 2174 "https": "443", 2175 "socks5": "1080", 2176 } 2177 2178 // canonicalAddr returns url.Host but always with a ":port" suffix 2179 func canonicalAddr(url *url.URL) string { 2180 addr := url.Hostname() 2181 if v, err := idnaASCII(addr); err == nil { 2182 addr = v 2183 } 2184 port := url.Port() 2185 if port == "" { 2186 port = portMap[url.Scheme] 2187 } 2188 return net.JoinHostPort(addr, port) 2189 } 2190 2191 // bodyEOFSignal is used by the HTTP/1 transport when reading response 2192 // bodies to make sure we see the end of a response body before 2193 // proceeding and reading on the connection again. 2194 // 2195 // It wraps a ReadCloser but runs fn (if non-nil) at most 2196 // once, right before its final (error-producing) Read or Close call 2197 // returns. fn should return the new error to return from Read or Close. 2198 // 2199 // If earlyCloseFn is non-nil and Close is called before io.EOF is 2200 // seen, earlyCloseFn is called instead of fn, and its return value is 2201 // the return value from Close. 2202 type bodyEOFSignal struct { 2203 body io.ReadCloser 2204 mu sync.Mutex // guards following 4 fields 2205 closed bool // whether Close has been called 2206 rerr error // sticky Read error 2207 fn func(error) error // err will be nil on Read io.EOF 2208 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 2209 } 2210 2211 var errReadOnClosedResBody = errors.New("http: read on closed response body") 2212 2213 func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 2214 es.mu.Lock() 2215 closed, rerr := es.closed, es.rerr 2216 es.mu.Unlock() 2217 if closed { 2218 return 0, errReadOnClosedResBody 2219 } 2220 if rerr != nil { 2221 return 0, rerr 2222 } 2223 2224 n, err = es.body.Read(p) 2225 if err != nil { 2226 es.mu.Lock() 2227 defer es.mu.Unlock() 2228 if es.rerr == nil { 2229 es.rerr = err 2230 } 2231 err = es.condfn(err) 2232 } 2233 return 2234 } 2235 2236 func (es *bodyEOFSignal) Close() error { 2237 es.mu.Lock() 2238 defer es.mu.Unlock() 2239 if es.closed { 2240 return nil 2241 } 2242 es.closed = true 2243 if es.earlyCloseFn != nil && es.rerr != io.EOF { 2244 return es.earlyCloseFn() 2245 } 2246 err := es.body.Close() 2247 return es.condfn(err) 2248 } 2249 2250 // caller must hold es.mu. 2251 func (es *bodyEOFSignal) condfn(err error) error { 2252 if es.fn == nil { 2253 return err 2254 } 2255 err = es.fn(err) 2256 es.fn = nil 2257 return err 2258 } 2259 2260 // gzipReader wraps a response body so it can lazily 2261 // call gzip.NewReader on the first call to Read 2262 type gzipReader struct { 2263 body *bodyEOFSignal // underlying HTTP/1 response body framing 2264 zr *gzip.Reader // lazily-initialized gzip reader 2265 zerr error // any error from gzip.NewReader; sticky 2266 } 2267 2268 func (gz *gzipReader) Read(p []byte) (n int, err error) { 2269 if gz.zr == nil { 2270 if gz.zerr == nil { 2271 gz.zr, gz.zerr = gzip.NewReader(gz.body) 2272 } 2273 if gz.zerr != nil { 2274 return 0, gz.zerr 2275 } 2276 } 2277 2278 gz.body.mu.Lock() 2279 if gz.body.closed { 2280 err = errReadOnClosedResBody 2281 } 2282 gz.body.mu.Unlock() 2283 2284 if err != nil { 2285 return 0, err 2286 } 2287 return gz.zr.Read(p) 2288 } 2289 2290 func (gz *gzipReader) Close() error { 2291 return gz.body.Close() 2292 } 2293 2294 type readerAndCloser struct { 2295 io.Reader 2296 io.Closer 2297 } 2298 2299 type tlsHandshakeTimeoutError struct{} 2300 2301 func (tlsHandshakeTimeoutError) Timeout() bool { return true } 2302 func (tlsHandshakeTimeoutError) Temporary() bool { return true } 2303 func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 2304 2305 // fakeLocker is a sync.Locker which does nothing. It's used to guard 2306 // test-only fields when not under test, to avoid runtime atomic 2307 // overhead. 2308 type fakeLocker struct{} 2309 2310 func (fakeLocker) Lock() {} 2311 func (fakeLocker) Unlock() {} 2312 2313 // clneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if 2314 // cfg is nil. This is safe to call even if cfg is in active use by a TLS 2315 // client or server. 2316 func cloneTLSConfig(cfg *tls.Config) *tls.Config { 2317 if cfg == nil { 2318 return &tls.Config{} 2319 } 2320 return cfg.Clone() 2321 } 2322 2323 type connLRU struct { 2324 ll *list.List // list.Element.Value type is of *persistConn 2325 m map[*persistConn]*list.Element 2326 } 2327 2328 // add adds pc to the head of the linked list. 2329 func (cl *connLRU) add(pc *persistConn) { 2330 if cl.ll == nil { 2331 cl.ll = list.New() 2332 cl.m = make(map[*persistConn]*list.Element) 2333 } 2334 ele := cl.ll.PushFront(pc) 2335 if _, ok := cl.m[pc]; ok { 2336 panic("persistConn was already in LRU") 2337 } 2338 cl.m[pc] = ele 2339 } 2340 2341 func (cl *connLRU) removeOldest() *persistConn { 2342 ele := cl.ll.Back() 2343 pc := ele.Value.(*persistConn) 2344 cl.ll.Remove(ele) 2345 delete(cl.m, pc) 2346 return pc 2347 } 2348 2349 // remove removes pc from cl. 2350 func (cl *connLRU) remove(pc *persistConn) { 2351 if ele, ok := cl.m[pc]; ok { 2352 cl.ll.Remove(ele) 2353 delete(cl.m, pc) 2354 } 2355 } 2356 2357 // len returns the number of items in the cache. 2358 func (cl *connLRU) len() int { 2359 return len(cl.m) 2360 } 2361 2362 // validPort reports whether p (without the colon) is a valid port in 2363 // a URL, per RFC 3986 Section 3.2.3, which says the port may be 2364 // empty, or only contain digits. 2365 func validPort(p string) bool { 2366 for _, r := range []byte(p) { 2367 if r < '0' || r > '9' { 2368 return false 2369 } 2370 } 2371 return true 2372 }