github.com/Kolosok86/http@v0.1.2/transport.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client implementation. See RFC 7230 through 7235. 6 // 7 // This is the low-level Transport implementation of RoundTripper. 8 // The high-level interface is in client.go. 9 10 package http 11 12 import ( 13 "bufio" 14 "compress/gzip" 15 "container/list" 16 "context" 17 "errors" 18 "fmt" 19 "io" 20 "log" 21 "net" 22 "net/url" 23 "os" 24 "reflect" 25 "strings" 26 "sync" 27 "sync/atomic" 28 "time" 29 30 "github.com/Kolosok86/http/httptrace" 31 "github.com/Kolosok86/http/internal/ascii" 32 "github.com/Kolosok86/http/textproto" 33 tls "github.com/refraction-networking/utls" 34 "golang.org/x/net/http/httpguts" 35 "golang.org/x/net/http/httpproxy" 36 ) 37 38 // DefaultTransport is the default implementation of Transport and is 39 // used by DefaultClient. It establishes network connections as needed 40 // and caches them for reuse by subsequent calls. It uses HTTP proxies 41 // as directed by the environment variables HTTP_PROXY, HTTPS_PROXY 42 // and NO_PROXY (or the lowercase versions thereof). 43 var DefaultTransport RoundTripper = &Transport{ 44 Proxy: ProxyFromEnvironment, 45 DialContext: defaultTransportDialContext(&net.Dialer{ 46 Timeout: 30 * time.Second, 47 KeepAlive: 30 * time.Second, 48 }), 49 ForceAttemptHTTP2: true, 50 MaxIdleConns: 100, 51 IdleConnTimeout: 90 * time.Second, 52 TLSHandshakeTimeout: 10 * time.Second, 53 ExpectContinueTimeout: 1 * time.Second, 54 } 55 56 // DefaultMaxIdleConnsPerHost is the default value of Transport's 57 // MaxIdleConnsPerHost. 58 const DefaultMaxIdleConnsPerHost = 2 59 60 // Transport is an implementation of RoundTripper that supports HTTP, 61 // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 62 // 63 // By default, Transport caches connections for future re-use. 64 // This may leave many open connections when accessing many hosts. 65 // This behavior can be managed using Transport's CloseIdleConnections method 66 // and the MaxIdleConnsPerHost and DisableKeepAlives fields. 67 // 68 // Transports should be reused instead of created as needed. 69 // Transports are safe for concurrent use by multiple goroutines. 70 // 71 // A Transport is a low-level primitive for making HTTP and HTTPS requests. 72 // For high-level functionality, such as cookies and redirects, see Client. 73 // 74 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 75 // for HTTPS URLs, depending on whether the server supports HTTP/2, 76 // and how the Transport is configured. The DefaultTransport supports HTTP/2. 77 // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 78 // and call ConfigureTransport. See the package docs for more about HTTP/2. 79 // 80 // Responses with status codes in the 1xx range are either handled 81 // automatically (100 expect-continue) or ignored. The one 82 // exception is HTTP status code 101 (Switching Protocols), which is 83 // considered a terminal status and returned by RoundTrip. To see the 84 // ignored 1xx responses, use the httptrace trace package's 85 // ClientTrace.Got1xxResponse. 86 // 87 // Transport only retries a request upon encountering a network error 88 // if the request is idempotent and either has no body or has its 89 // Request.GetBody defined. HTTP requests are considered idempotent if 90 // they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their 91 // Header map contains an "Idempotency-Key" or "X-Idempotency-Key" 92 // entry. If the idempotency Key value is a zero-length slice, the 93 // request is treated as idempotent but the header is not sent on the 94 // wire. 95 type Transport struct { 96 idleMu sync.Mutex 97 closeIdle bool // user has requested to close all idle conns 98 idleConn map[connectMethodKey][]*persistConn // most recently used at end 99 idleConnWait map[connectMethodKey]wantConnQueue // waiting getConns 100 idleLRU connLRU 101 102 reqMu sync.Mutex 103 reqCanceler map[cancelKey]func(error) 104 105 altMu sync.Mutex // guards changing altProto only 106 altProto atomic.Value // of nil or map[string]RoundTripper, Key is URI scheme 107 108 connsPerHostMu sync.Mutex 109 connsPerHost map[connectMethodKey]int 110 connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns 111 112 // Proxy specifies a function to return a proxy for a given 113 // Request. If the function returns a non-nil error, the 114 // request is aborted with the provided error. 115 // 116 // The proxy type is determined by the URL scheme. "http", 117 // "https", and "socks5" are supported. If the scheme is empty, 118 // "http" is assumed. 119 // 120 // If Proxy is nil or returns a nil *URL, no proxy is used. 121 Proxy func(*Request) (*url.URL, error) 122 123 // OnProxyConnectResponse is called when the Transport gets an HTTP response from 124 // a proxy for a CONNECT request. It's called before the check for a 200 OK response. 125 // If it returns an error, the request fails with that error. 126 OnProxyConnectResponse func(ctx context.Context, proxyURL *url.URL, connectReq *Request, connectRes *Response) error 127 128 // DialContext specifies the dial function for creating unencrypted TCP connections. 129 // If DialContext is nil (and the deprecated Dial below is also nil), 130 // then the transport dials using package net. 131 // 132 // DialContext runs concurrently with calls to RoundTrip. 133 // A RoundTrip call that initiates a dial may end up using 134 // a connection dialed previously when the earlier connection 135 // becomes idle before the later DialContext completes. 136 DialContext func(ctx context.Context, network, addr string) (net.Conn, error) 137 138 // Dial specifies the dial function for creating unencrypted TCP connections. 139 // 140 // Dial runs concurrently with calls to RoundTrip. 141 // A RoundTrip call that initiates a dial may end up using 142 // a connection dialed previously when the earlier connection 143 // becomes idle before the later Dial completes. 144 // 145 // Deprecated: Use DialContext instead, which allows the transport 146 // to cancel dials as soon as they are no longer needed. 147 // If both are set, DialContext takes priority. 148 Dial func(network, addr string) (net.Conn, error) 149 150 // DialTLSContext specifies an optional dial function for creating 151 // TLS connections for non-proxied HTTPS requests. 152 // 153 // If DialTLSContext is nil (and the deprecated DialTLS below is also nil), 154 // DialContext and TLSClientConfig are used. 155 // 156 // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS 157 // requests and the TLSClientConfig and TLSHandshakeTimeout 158 // are ignored. The returned net.Conn is assumed to already be 159 // past the TLS handshake. 160 DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) 161 162 // DialTLS specifies an optional dial function for creating 163 // TLS connections for non-proxied HTTPS requests. 164 // 165 // Deprecated: Use DialTLSContext instead, which allows the transport 166 // to cancel dials as soon as they are no longer needed. 167 // If both are set, DialTLSContext takes priority. 168 DialTLS func(network, addr string) (net.Conn, error) 169 170 // TLSClientConfig specifies the TLS configuration to use with 171 // tls.Client. 172 // If nil, the default configuration is used. 173 // If non-nil, HTTP/2 support may not be enabled by default. 174 TLSClientConfig *tls.Config 175 176 // TLSHandshakeTimeout specifies the maximum amount of time waiting to 177 // wait for a TLS handshake. Zero means no timeout. 178 TLSHandshakeTimeout time.Duration 179 180 // DisableKeepAlives, if true, disables HTTP keep-alives and 181 // will only use the connection to the server for a single 182 // HTTP request. 183 // 184 // This is unrelated to the similarly named TCP keep-alives. 185 DisableKeepAlives bool 186 187 // DisableCompression, if true, prevents the Transport from 188 // requesting compression with an "Accept-Encoding: gzip" 189 // request header when the Request contains no existing 190 // Accept-Encoding value. If the Transport requests gzip on 191 // its own and gets a gzipped response, it's transparently 192 // decoded in the Response.Body. However, if the user 193 // explicitly requested gzip it is not automatically 194 // uncompressed. 195 DisableCompression bool 196 197 // MaxIdleConns controls the maximum number of idle (keep-alive) 198 // connections across all hosts. Zero means no limit. 199 MaxIdleConns int 200 201 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 202 // (keep-alive) connections to keep per-host. If zero, 203 // DefaultMaxIdleConnsPerHost is used. 204 MaxIdleConnsPerHost int 205 206 // MaxConnsPerHost optionally limits the total number of 207 // connections per host, including connections in the dialing, 208 // active, and idle states. On limit violation, dials will block. 209 // 210 // Zero means no limit. 211 MaxConnsPerHost int 212 213 // IdleConnTimeout is the maximum amount of time an idle 214 // (keep-alive) connection will remain idle before closing 215 // itself. 216 // Zero means no limit. 217 IdleConnTimeout time.Duration 218 219 // ResponseHeaderTimeout, if non-zero, specifies the amount of 220 // time to wait for a server's response headers after fully 221 // writing the request (including its body, if any). This 222 // time does not include the time to read the response body. 223 ResponseHeaderTimeout time.Duration 224 225 // ExpectContinueTimeout, if non-zero, specifies the amount of 226 // time to wait for a server's first response headers after fully 227 // writing the request headers if the request has an 228 // "Expect: 100-continue" header. Zero means no timeout and 229 // causes the body to be sent immediately, without 230 // waiting for the server to approve. 231 // This time does not include the time to send the request header. 232 ExpectContinueTimeout time.Duration 233 234 // TLSNextProto specifies how the Transport switches to an 235 // alternate protocol (such as HTTP/2) after a TLS ALPN 236 // protocol negotiation. If Transport dials an TLS connection 237 // with a non-empty protocol name and TLSNextProto contains a 238 // map entry for that Key (such as "h2"), then the func is 239 // called with the request's authority (such as "example.com" 240 // or "example.com:1234") and the TLS connection. The function 241 // must return a RoundTripper that then handles the request. 242 // If TLSNextProto is not nil, HTTP/2 support is not enabled 243 // automatically. 244 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 245 246 // ProxyConnectHeader optionally specifies headers to send to 247 // proxies during CONNECT requests. 248 // To set the header dynamically, see GetProxyConnectHeader. 249 ProxyConnectHeader Header 250 251 // GetProxyConnectHeader optionally specifies a func to return 252 // headers to send to proxyURL during a CONNECT request to the 253 // ip:port target. 254 // If it returns an error, the Transport's RoundTrip fails with 255 // that error. It can return (nil, nil) to not add headers. 256 // If GetProxyConnectHeader is non-nil, ProxyConnectHeader is 257 // ignored. 258 GetProxyConnectHeader func(ctx context.Context, proxyURL *url.URL, target string) (Header, error) 259 260 // MaxResponseHeaderBytes specifies a limit on how many 261 // response bytes are allowed in the server's response 262 // header. 263 // 264 // Zero means to use a default limit. 265 MaxResponseHeaderBytes int64 266 267 // WriteBufferSize specifies the size of the write buffer used 268 // when writing to the transport. 269 // If zero, a default (currently 4KB) is used. 270 WriteBufferSize int 271 272 // ReadBufferSize specifies the size of the read buffer used 273 // when reading from the transport. 274 // If zero, a default (currently 4KB) is used. 275 ReadBufferSize int 276 277 // nextProtoOnce guards initialization of TLSNextProto and 278 // h2transport (via onceSetNextProtoDefaults) 279 nextProtoOnce sync.Once 280 h2transport h2Transport // non-nil if http2 wired up 281 tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired 282 283 // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero 284 // Dial, DialTLS, or DialContext func or TLSClientConfig is provided. 285 // By default, use of any those fields conservatively disables HTTP/2. 286 // To use a custom dialer or TLS config and still attempt HTTP/2 287 // upgrades, set this to true. 288 ForceAttemptHTTP2 bool 289 } 290 291 // A cancelKey is the Key of the reqCanceler map. 292 // We wrap the *Request in this type since we want to use the original request, 293 // not any transient one created by roundTrip. 294 type cancelKey struct { 295 req *Request 296 } 297 298 func (t *Transport) writeBufferSize() int { 299 if t.WriteBufferSize > 0 { 300 return t.WriteBufferSize 301 } 302 return 4 << 10 303 } 304 305 func (t *Transport) readBufferSize() int { 306 if t.ReadBufferSize > 0 { 307 return t.ReadBufferSize 308 } 309 return 4 << 10 310 } 311 312 // Clone returns a deep copy of t's exported fields. 313 func (t *Transport) Clone() *Transport { 314 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 315 t2 := &Transport{ 316 Proxy: t.Proxy, 317 OnProxyConnectResponse: t.OnProxyConnectResponse, 318 DialContext: t.DialContext, 319 Dial: t.Dial, 320 DialTLS: t.DialTLS, 321 DialTLSContext: t.DialTLSContext, 322 TLSHandshakeTimeout: t.TLSHandshakeTimeout, 323 DisableKeepAlives: t.DisableKeepAlives, 324 DisableCompression: t.DisableCompression, 325 MaxIdleConns: t.MaxIdleConns, 326 MaxIdleConnsPerHost: t.MaxIdleConnsPerHost, 327 MaxConnsPerHost: t.MaxConnsPerHost, 328 IdleConnTimeout: t.IdleConnTimeout, 329 ResponseHeaderTimeout: t.ResponseHeaderTimeout, 330 ExpectContinueTimeout: t.ExpectContinueTimeout, 331 ProxyConnectHeader: t.ProxyConnectHeader.Clone(), 332 GetProxyConnectHeader: t.GetProxyConnectHeader, 333 MaxResponseHeaderBytes: t.MaxResponseHeaderBytes, 334 ForceAttemptHTTP2: t.ForceAttemptHTTP2, 335 WriteBufferSize: t.WriteBufferSize, 336 ReadBufferSize: t.ReadBufferSize, 337 } 338 if t.TLSClientConfig != nil { 339 t2.TLSClientConfig = t.TLSClientConfig.Clone() 340 } 341 if !t.tlsNextProtoWasNil { 342 npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} 343 for k, v := range t.TLSNextProto { 344 npm[k] = v 345 } 346 t2.TLSNextProto = npm 347 } 348 return t2 349 } 350 351 // h2Transport is the interface we expect to be able to call from 352 // net/http against an *http2.Transport that's either bundled into 353 // h2_bundle.go or supplied by the user via x/net/http2. 354 // 355 // We name it with the "h2" prefix to stay out of the "http2" prefix 356 // namespace used by x/tools/cmd/bundle for h2_bundle.go. 357 type h2Transport interface { 358 CloseIdleConnections() 359 } 360 361 func (t *Transport) hasCustomTLSDialer() bool { 362 return t.DialTLS != nil || t.DialTLSContext != nil 363 } 364 365 // onceSetNextProtoDefaults initializes TLSNextProto. 366 // It must be called via t.nextProtoOnce.Do. 367 func (t *Transport) onceSetNextProtoDefaults() { 368 t.tlsNextProtoWasNil = (t.TLSNextProto == nil) 369 if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") { 370 return 371 } 372 373 // If they've already configured http2 with 374 // golang.org/x/net/http2 instead of the bundled copy, try to 375 // get at its http2.Transport value (via the "https" 376 // altproto map) so we can call CloseIdleConnections on it if 377 // requested. (Issue 22891) 378 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 379 if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 { 380 if v := rv.Field(0); v.CanInterface() { 381 if h2i, ok := v.Interface().(h2Transport); ok { 382 t.h2transport = h2i 383 return 384 } 385 } 386 } 387 388 if t.TLSNextProto != nil { 389 // This is the documented way to disable http2 on a 390 // Transport. 391 return 392 } 393 if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { 394 // Be conservative and don't automatically enable 395 // http2 if they've specified a custom TLS config or 396 // custom dialers. Let them opt-in themselves via 397 // http2.ConfigureTransport so we don't surprise them 398 // by modifying their tls.Config. Issue 14275. 399 // However, if ForceAttemptHTTP2 is true, it overrides the above checks. 400 return 401 } 402 if omitBundledHTTP2 { 403 return 404 } 405 t2, err := http2configureTransports(t) 406 if err != nil { 407 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 408 return 409 } 410 t.h2transport = t2 411 412 // Auto-configure the http2.Transport's MaxHeaderListSize from 413 // the http.Transport's MaxResponseHeaderBytes. They don't 414 // exactly mean the same thing, but they're close. 415 // 416 // TODO: also add this to x/net/http2.Configure Transport, behind 417 // a +build go1.7 build tag: 418 if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { 419 const h2max = 1<<32 - 1 420 if limit1 >= h2max { 421 t2.MaxHeaderListSize = h2max 422 } else { 423 t2.MaxHeaderListSize = uint32(limit1) 424 } 425 } 426 } 427 428 // ProxyFromEnvironment returns the URL of the proxy to use for a 429 // given request, as indicated by the environment variables 430 // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 431 // thereof). Requests use the proxy from the environment variable 432 // matching their scheme, unless excluded by NO_PROXY. 433 // 434 // The environment Values may be either a complete URL or a 435 // "host[:port]", in which case the "http" scheme is assumed. 436 // The schemes "http", "https", and "socks5" are supported. 437 // An error is returned if the value is a different form. 438 // 439 // A nil URL and nil error are returned if no proxy is defined in the 440 // environment, or a proxy should not be used for the given request, 441 // as defined by NO_PROXY. 442 // 443 // As a special case, if req.URL.Host is "localhost" (with or without 444 // a port number), then a nil URL and nil error will be returned. 445 func ProxyFromEnvironment(req *Request) (*url.URL, error) { 446 return envProxyFunc()(req.URL) 447 } 448 449 // ProxyURL returns a proxy function (for use in a Transport) 450 // that always returns the same URL. 451 func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 452 return func(*Request) (*url.URL, error) { 453 return fixedURL, nil 454 } 455 } 456 457 // transportRequest is a wrapper around a *Request that adds 458 // optional extra headers to write and stores any error to return 459 // from roundTrip. 460 type transportRequest struct { 461 *Request // original request, not to be mutated 462 extra Header // extra headers to write, or nil 463 trace *httptrace.ClientTrace // optional 464 cancelKey cancelKey 465 466 mu sync.Mutex // guards err 467 err error // first setError value for mapRoundTripError to consider 468 } 469 470 func (tr *transportRequest) extraHeaders() Header { 471 if tr.extra == nil { 472 tr.extra = make(Header) 473 } 474 return tr.extra 475 } 476 477 func (tr *transportRequest) setError(err error) { 478 tr.mu.Lock() 479 if tr.err == nil { 480 tr.err = err 481 } 482 tr.mu.Unlock() 483 } 484 485 // useRegisteredProtocol reports whether an alternate protocol (as registered 486 // with Transport.RegisterProtocol) should be respected for this request. 487 func (t *Transport) useRegisteredProtocol(req *Request) bool { 488 if req.URL.Scheme == "https" && req.requiresHTTP1() { 489 // If this request requires HTTP/1, don't use the 490 // "https" alternate protocol, which is used by the 491 // HTTP/2 code to take over requests if there's an 492 // existing cached HTTP/2 connection. 493 return false 494 } 495 return true 496 } 497 498 // alternateRoundTripper returns the alternate RoundTripper to use 499 // for this request if the Request's URL scheme requires one, 500 // or nil for the normal case of using the Transport. 501 func (t *Transport) alternateRoundTripper(req *Request) RoundTripper { 502 if !t.useRegisteredProtocol(req) { 503 return nil 504 } 505 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 506 return altProto[req.URL.Scheme] 507 } 508 509 // roundTrip implements a RoundTripper over HTTP. 510 func (t *Transport) roundTrip(req *Request) (*Response, error) { 511 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 512 ctx := req.Context() 513 trace := httptrace.ContextClientTrace(ctx) 514 515 if req.URL == nil { 516 req.closeBody() 517 return nil, errors.New("http: nil Request.URL") 518 } 519 if req.Header == nil { 520 req.closeBody() 521 return nil, errors.New("http: nil Request.Header") 522 } 523 scheme := req.URL.Scheme 524 isHTTP := scheme == "http" || scheme == "https" 525 if isHTTP { 526 for k, vv := range req.Header { 527 if !httpguts.ValidHeaderFieldName(k) { 528 req.closeBody() 529 return nil, fmt.Errorf("net/http: invalid header field name %q", k) 530 } 531 for _, v := range vv { 532 if !httpguts.ValidHeaderFieldValue(v) { 533 req.closeBody() 534 // Don't include the value in the error, because it may be sensitive. 535 return nil, fmt.Errorf("net/http: invalid header field value for %q", k) 536 } 537 } 538 } 539 } 540 541 origReq := req 542 cancelKey := cancelKey{origReq} 543 req = setupRewindBody(req) 544 545 if altRT := t.alternateRoundTripper(req); altRT != nil { 546 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 547 return resp, err 548 } 549 var err error 550 req, err = rewindBody(req) 551 if err != nil { 552 return nil, err 553 } 554 } 555 if !isHTTP { 556 req.closeBody() 557 return nil, badStringError("unsupported protocol scheme", scheme) 558 } 559 if req.Method != "" && !validMethod(req.Method) { 560 req.closeBody() 561 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 562 } 563 if req.URL.Host == "" { 564 req.closeBody() 565 return nil, errors.New("http: no Host in request URL") 566 } 567 568 for { 569 select { 570 case <-ctx.Done(): 571 req.closeBody() 572 return nil, ctx.Err() 573 default: 574 } 575 576 // treq gets modified by roundTrip, so we need to recreate for each retry. 577 treq := &transportRequest{Request: req, trace: trace, cancelKey: cancelKey} 578 cm, err := t.connectMethodForRequest(treq) 579 if err != nil { 580 req.closeBody() 581 return nil, err 582 } 583 584 // Get the cached or newly-created connection to either the 585 // host (for http or https), the http proxy, or the http proxy 586 // pre-CONNECTed to https server. In any case, we'll be ready 587 // to send it requests. 588 pconn, err := t.getConn(treq, cm) 589 if err != nil { 590 t.setReqCanceler(cancelKey, nil) 591 req.closeBody() 592 return nil, err 593 } 594 595 var resp *Response 596 if pconn.alt != nil { 597 // HTTP/2 path. 598 t.setReqCanceler(cancelKey, nil) // not cancelable with CancelRequest 599 resp, err = pconn.alt.RoundTrip(req) 600 } else { 601 resp, err = pconn.roundTrip(treq) 602 } 603 if err == nil { 604 resp.Request = origReq 605 return resp, nil 606 } 607 608 // Failed. Clean up and determine whether to retry. 609 if http2isNoCachedConnError(err) { 610 if t.removeIdleConn(pconn) { 611 t.decConnsPerHost(pconn.cacheKey) 612 } 613 } else if !pconn.shouldRetryRequest(req, err) { 614 // Issue 16465: return underlying net.Conn.Read error from peek, 615 // as we've historically done. 616 if e, ok := err.(nothingWrittenError); ok { 617 err = e.error 618 } 619 if e, ok := err.(transportReadFromServerError); ok { 620 err = e.err 621 } 622 if b, ok := req.Body.(*readTrackingBody); ok && !b.didClose { 623 // Issue 49621: Close the request body if pconn.roundTrip 624 // didn't do so already. This can happen if the pconn 625 // write loop exits without reading the write request. 626 req.closeBody() 627 } 628 return nil, err 629 } 630 testHookRoundTripRetried() 631 632 // Rewind the body if we're able to. 633 req, err = rewindBody(req) 634 if err != nil { 635 return nil, err 636 } 637 } 638 } 639 640 var errCannotRewind = errors.New("net/http: cannot rewind body after connection loss") 641 642 type readTrackingBody struct { 643 io.ReadCloser 644 didRead bool 645 didClose bool 646 } 647 648 func (r *readTrackingBody) Read(data []byte) (int, error) { 649 r.didRead = true 650 return r.ReadCloser.Read(data) 651 } 652 653 func (r *readTrackingBody) Close() error { 654 r.didClose = true 655 return r.ReadCloser.Close() 656 } 657 658 // setupRewindBody returns a new request with a custom body wrapper 659 // that can report whether the body needs rewinding. 660 // This lets rewindBody avoid an error result when the request 661 // does not have GetBody but the body hasn't been read at all yet. 662 func setupRewindBody(req *Request) *Request { 663 if req.Body == nil || req.Body == NoBody { 664 return req 665 } 666 newReq := *req 667 newReq.Body = &readTrackingBody{ReadCloser: req.Body} 668 return &newReq 669 } 670 671 // rewindBody returns a new request with the body rewound. 672 // It returns req unmodified if the body does not need rewinding. 673 // rewindBody takes care of closing req.Body when appropriate 674 // (in all cases except when rewindBody returns req unmodified). 675 func rewindBody(req *Request) (rewound *Request, err error) { 676 if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose) { 677 return req, nil // nothing to rewind 678 } 679 if !req.Body.(*readTrackingBody).didClose { 680 req.closeBody() 681 } 682 if req.GetBody == nil { 683 return nil, errCannotRewind 684 } 685 body, err := req.GetBody() 686 if err != nil { 687 return nil, err 688 } 689 newReq := *req 690 newReq.Body = &readTrackingBody{ReadCloser: body} 691 return &newReq, nil 692 } 693 694 // shouldRetryRequest reports whether we should retry sending a failed 695 // HTTP request on a new connection. The non-nil input error is the 696 // error from roundTrip. 697 func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { 698 if http2isNoCachedConnError(err) { 699 // Issue 16582: if the user started a bunch of 700 // requests at once, they can all pick the same conn 701 // and violate the server's max concurrent streams. 702 // Instead, match the HTTP/1 behavior for now and dial 703 // again to get a new TCP connection, rather than failing 704 // this request. 705 return true 706 } 707 if err == errMissingHost { 708 // User error. 709 return false 710 } 711 if !pc.isReused() { 712 // This was a fresh connection. There's no reason the server 713 // should've hung up on us. 714 // 715 // Also, if we retried now, we could loop forever 716 // creating new connections and retrying if the server 717 // is just hanging up on us because it doesn't like 718 // our request (as opposed to sending an error). 719 return false 720 } 721 if _, ok := err.(nothingWrittenError); ok { 722 // We never wrote anything, so it's safe to retry, if there's no body or we 723 // can "rewind" the body with GetBody. 724 return req.outgoingLength() == 0 || req.GetBody != nil 725 } 726 if !req.isReplayable() { 727 // Don't retry non-idempotent requests. 728 return false 729 } 730 if _, ok := err.(transportReadFromServerError); ok { 731 // We got some non-EOF net.Conn.Read failure reading 732 // the 1st response byte from the server. 733 return true 734 } 735 if err == errServerClosedIdle { 736 // The server replied with io.EOF while we were trying to 737 // read the response. Probably an unfortunately keep-alive 738 // timeout, just as the client was writing a request. 739 return true 740 } 741 return false // conservatively 742 } 743 744 // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 745 var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 746 747 // RegisterProtocol registers a new protocol with scheme. 748 // The Transport will pass requests using the given scheme to rt. 749 // It is rt's responsibility to simulate HTTP request semantics. 750 // 751 // RegisterProtocol can be used by other packages to provide 752 // implementations of protocol schemes like "ftp" or "file". 753 // 754 // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will 755 // handle the RoundTrip itself for that one request, as if the 756 // protocol were not registered. 757 func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 758 t.altMu.Lock() 759 defer t.altMu.Unlock() 760 oldMap, _ := t.altProto.Load().(map[string]RoundTripper) 761 if _, exists := oldMap[scheme]; exists { 762 panic("protocol " + scheme + " already registered") 763 } 764 newMap := make(map[string]RoundTripper) 765 for k, v := range oldMap { 766 newMap[k] = v 767 } 768 newMap[scheme] = rt 769 t.altProto.Store(newMap) 770 } 771 772 // CloseIdleConnections closes any connections which were previously 773 // connected from previous requests but are now sitting idle in 774 // a "keep-alive" state. It does not interrupt any connections currently 775 // in use. 776 func (t *Transport) CloseIdleConnections() { 777 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 778 t.idleMu.Lock() 779 m := t.idleConn 780 t.idleConn = nil 781 t.closeIdle = true // close newly idle connections 782 t.idleLRU = connLRU{} 783 t.idleMu.Unlock() 784 for _, conns := range m { 785 for _, pconn := range conns { 786 pconn.close(errCloseIdleConns) 787 } 788 } 789 if t2 := t.h2transport; t2 != nil { 790 t2.CloseIdleConnections() 791 } 792 } 793 794 // CancelRequest cancels an in-flight request by closing its connection. 795 // CancelRequest should only be called after RoundTrip has returned. 796 // 797 // Deprecated: Use Request.WithContext to create a request with a 798 // cancelable context instead. CancelRequest cannot cancel HTTP/2 799 // requests. 800 func (t *Transport) CancelRequest(req *Request) { 801 t.cancelRequest(cancelKey{req}, errRequestCanceled) 802 } 803 804 // Cancel an in-flight request, recording the error value. 805 // Returns whether the request was canceled. 806 func (t *Transport) cancelRequest(key cancelKey, err error) bool { 807 // This function must not return until the cancel func has completed. 808 // See: https://golang.org/issue/34658 809 t.reqMu.Lock() 810 defer t.reqMu.Unlock() 811 cancel := t.reqCanceler[key] 812 delete(t.reqCanceler, key) 813 if cancel != nil { 814 cancel(err) 815 } 816 817 return cancel != nil 818 } 819 820 // 821 // Private implementation past this point. 822 // 823 824 var ( 825 envProxyOnce sync.Once 826 envProxyFuncValue func(*url.URL) (*url.URL, error) 827 ) 828 829 // envProxyFunc returns a function that reads the 830 // environment variable to determine the proxy address. 831 func envProxyFunc() func(*url.URL) (*url.URL, error) { 832 envProxyOnce.Do(func() { 833 envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() 834 }) 835 return envProxyFuncValue 836 } 837 838 // resetProxyConfig is used by tests. 839 func resetProxyConfig() { 840 envProxyOnce = sync.Once{} 841 envProxyFuncValue = nil 842 } 843 844 func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 845 cm.targetScheme = treq.URL.Scheme 846 cm.targetAddr = canonicalAddr(treq.URL) 847 if t.Proxy != nil { 848 cm.proxyURL, err = t.Proxy(treq.Request) 849 } 850 cm.onlyH1 = treq.requiresHTTP1() 851 return cm, err 852 } 853 854 // proxyAuth returns the Proxy-Authorization header to set 855 // on requests, if applicable. 856 func (cm *connectMethod) proxyAuth() string { 857 if cm.proxyURL == nil { 858 return "" 859 } 860 if u := cm.proxyURL.User; u != nil { 861 username := u.Username() 862 password, _ := u.Password() 863 return "Basic " + basicAuth(username, password) 864 } 865 return "" 866 } 867 868 // error Values for debugging and testing, not seen by users. 869 var ( 870 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") 871 errConnBroken = errors.New("http: putIdleConn: connection is in bad state") 872 errCloseIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") 873 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") 874 errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") 875 errCloseIdleConns = errors.New("http: CloseIdleConnections called") 876 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") 877 errIdleConnTimeout = errors.New("http: idle connection timeout") 878 879 // errServerClosedIdle is not seen by users for idempotent requests, but may be 880 // seen by a user if the server shuts down an idle connection and sends its FIN 881 // in flight with already-written POST body bytes from the client. 882 // See https://github.com/golang/go/issues/19943#issuecomment-355607646 883 errServerClosedIdle = errors.New("http: server closed idle connection") 884 ) 885 886 // transportReadFromServerError is used by Transport.readLoop when the 887 // 1 byte peek read fails and we're actually anticipating a response. 888 // Usually this is just due to the inherent keep-alive shut down race, 889 // where the server closed the connection at the same time the client 890 // wrote. The underlying err field is usually io.EOF or some 891 // ECONNRESET sort of thing which varies by platform. But it might be 892 // the user's custom net.Conn.Read error too, so we carry it along for 893 // them to return from Transport.RoundTrip. 894 type transportReadFromServerError struct { 895 err error 896 } 897 898 func (e transportReadFromServerError) Unwrap() error { return e.err } 899 900 func (e transportReadFromServerError) Error() string { 901 return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) 902 } 903 904 func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { 905 if err := t.tryPutIdleConn(pconn); err != nil { 906 pconn.close(err) 907 } 908 } 909 910 func (t *Transport) maxIdleConnsPerHost() int { 911 if v := t.MaxIdleConnsPerHost; v != 0 { 912 return v 913 } 914 return DefaultMaxIdleConnsPerHost 915 } 916 917 // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting 918 // a new request. 919 // If pconn is no longer needed or not in a good state, tryPutIdleConn returns 920 // an error explaining why it wasn't registered. 921 // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. 922 func (t *Transport) tryPutIdleConn(pconn *persistConn) error { 923 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 924 return errKeepAlivesDisabled 925 } 926 if pconn.isBroken() { 927 return errConnBroken 928 } 929 pconn.markReused() 930 931 t.idleMu.Lock() 932 defer t.idleMu.Unlock() 933 934 // HTTP/2 (pconn.alt != nil) connections do not come out of the idle list, 935 // because multiple goroutines can use them simultaneously. 936 // If this is an HTTP/2 connection being “returned,” we're done. 937 if pconn.alt != nil && t.idleLRU.m[pconn] != nil { 938 return nil 939 } 940 941 // Deliver pconn to goroutine waiting for idle connection, if any. 942 // (They may be actively dialing, but this conn is ready first. 943 // Chrome calls this socket late binding. 944 // See https://www.chromium.org/developers/design-documents/network-stack#TOC-Connection-Management.) 945 key := pconn.cacheKey 946 if q, ok := t.idleConnWait[key]; ok { 947 done := false 948 if pconn.alt == nil { 949 // HTTP/1. 950 // Loop over the waiting list until we find a w that isn't done already, and hand it pconn. 951 for q.len() > 0 { 952 w := q.popFront() 953 if w.tryDeliver(pconn, nil) { 954 done = true 955 break 956 } 957 } 958 } else { 959 // HTTP/2. 960 // Can hand the same pconn to everyone in the waiting list, 961 // and we still won't be done: we want to put it in the idle 962 // list unconditionally, for any future clients too. 963 for q.len() > 0 { 964 w := q.popFront() 965 w.tryDeliver(pconn, nil) 966 } 967 } 968 if q.len() == 0 { 969 delete(t.idleConnWait, key) 970 } else { 971 t.idleConnWait[key] = q 972 } 973 if done { 974 return nil 975 } 976 } 977 978 if t.closeIdle { 979 return errCloseIdle 980 } 981 if t.idleConn == nil { 982 t.idleConn = make(map[connectMethodKey][]*persistConn) 983 } 984 idles := t.idleConn[key] 985 if len(idles) >= t.maxIdleConnsPerHost() { 986 return errTooManyIdleHost 987 } 988 for _, exist := range idles { 989 if exist == pconn { 990 log.Fatalf("dup idle pconn %p in freelist", pconn) 991 } 992 } 993 t.idleConn[key] = append(idles, pconn) 994 t.idleLRU.add(pconn) 995 if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { 996 oldest := t.idleLRU.removeOldest() 997 oldest.close(errTooManyIdle) 998 t.removeIdleConnLocked(oldest) 999 } 1000 1001 // Set idle timer, but only for HTTP/1 (pconn.alt == nil). 1002 // The HTTP/2 implementation manages the idle timer itself 1003 // (see idleConnTimeout in h2_bundle.go). 1004 if t.IdleConnTimeout > 0 && pconn.alt == nil { 1005 if pconn.idleTimer != nil { 1006 pconn.idleTimer.Reset(t.IdleConnTimeout) 1007 } else { 1008 pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) 1009 } 1010 } 1011 pconn.idleAt = time.Now() 1012 return nil 1013 } 1014 1015 // queueForIdleConn queues w to receive the next idle connection for w.cm. 1016 // As an optimization hint to the caller, queueForIdleConn reports whether 1017 // it successfully delivered an already-idle connection. 1018 func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) { 1019 if t.DisableKeepAlives { 1020 return false 1021 } 1022 1023 t.idleMu.Lock() 1024 defer t.idleMu.Unlock() 1025 1026 // Stop closing connections that become idle - we might want one. 1027 // (That is, undo the effect of t.CloseIdleConnections.) 1028 t.closeIdle = false 1029 1030 if w == nil { 1031 // Happens in test hook. 1032 return false 1033 } 1034 1035 // If IdleConnTimeout is set, calculate the oldest 1036 // persistConn.idleAt time we're willing to use a cached idle 1037 // conn. 1038 var oldTime time.Time 1039 if t.IdleConnTimeout > 0 { 1040 oldTime = time.Now().Add(-t.IdleConnTimeout) 1041 } 1042 1043 // Look for most recently-used idle connection. 1044 if list, ok := t.idleConn[w.key]; ok { 1045 stop := false 1046 delivered := false 1047 for len(list) > 0 && !stop { 1048 pconn := list[len(list)-1] 1049 1050 // See whether this connection has been idle too long, considering 1051 // only the wall time (the Round(0)), in case this is a laptop or VM 1052 // coming out of suspend with previously cached idle connections. 1053 tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime) 1054 if tooOld { 1055 // Async cleanup. Launch in its own goroutine (as if a 1056 // time.AfterFunc called it); it acquires idleMu, which we're 1057 // holding, and does a synchronous net.Conn.Close. 1058 go pconn.closeConnIfStillIdle() 1059 } 1060 if pconn.isBroken() || tooOld { 1061 // If either persistConn.readLoop has marked the connection 1062 // broken, but Transport.removeIdleConn has not yet removed it 1063 // from the idle list, or if this persistConn is too old (it was 1064 // idle too long), then ignore it and look for another. In both 1065 // cases it's already in the process of being closed. 1066 list = list[:len(list)-1] 1067 continue 1068 } 1069 delivered = w.tryDeliver(pconn, nil) 1070 if delivered { 1071 if pconn.alt != nil { 1072 // HTTP/2: multiple clients can share pconn. 1073 // Leave it in the list. 1074 } else { 1075 // HTTP/1: only one client can use pconn. 1076 // Remove it from the list. 1077 t.idleLRU.remove(pconn) 1078 list = list[:len(list)-1] 1079 } 1080 } 1081 stop = true 1082 } 1083 if len(list) > 0 { 1084 t.idleConn[w.key] = list 1085 } else { 1086 delete(t.idleConn, w.key) 1087 } 1088 if stop { 1089 return delivered 1090 } 1091 } 1092 1093 // Register to receive next connection that becomes idle. 1094 if t.idleConnWait == nil { 1095 t.idleConnWait = make(map[connectMethodKey]wantConnQueue) 1096 } 1097 q := t.idleConnWait[w.key] 1098 q.cleanFront() 1099 q.pushBack(w) 1100 t.idleConnWait[w.key] = q 1101 return false 1102 } 1103 1104 // removeIdleConn marks pconn as dead. 1105 func (t *Transport) removeIdleConn(pconn *persistConn) bool { 1106 t.idleMu.Lock() 1107 defer t.idleMu.Unlock() 1108 return t.removeIdleConnLocked(pconn) 1109 } 1110 1111 // t.idleMu must be held. 1112 func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool { 1113 if pconn.idleTimer != nil { 1114 pconn.idleTimer.Stop() 1115 } 1116 t.idleLRU.remove(pconn) 1117 key := pconn.cacheKey 1118 pconns := t.idleConn[key] 1119 var removed bool 1120 switch len(pconns) { 1121 case 0: 1122 // Nothing 1123 case 1: 1124 if pconns[0] == pconn { 1125 delete(t.idleConn, key) 1126 removed = true 1127 } 1128 default: 1129 for i, v := range pconns { 1130 if v != pconn { 1131 continue 1132 } 1133 // Slide down, keeping most recently-used 1134 // conns at the end. 1135 copy(pconns[i:], pconns[i+1:]) 1136 t.idleConn[key] = pconns[:len(pconns)-1] 1137 removed = true 1138 break 1139 } 1140 } 1141 return removed 1142 } 1143 1144 func (t *Transport) setReqCanceler(key cancelKey, fn func(error)) { 1145 t.reqMu.Lock() 1146 defer t.reqMu.Unlock() 1147 if t.reqCanceler == nil { 1148 t.reqCanceler = make(map[cancelKey]func(error)) 1149 } 1150 if fn != nil { 1151 t.reqCanceler[key] = fn 1152 } else { 1153 delete(t.reqCanceler, key) 1154 } 1155 } 1156 1157 // replaceReqCanceler replaces an existing cancel function. If there is no cancel function 1158 // for the request, we don't set the function and return false. 1159 // Since CancelRequest will clear the canceler, we can use the return value to detect if 1160 // the request was canceled since the last setReqCancel call. 1161 func (t *Transport) replaceReqCanceler(key cancelKey, fn func(error)) bool { 1162 t.reqMu.Lock() 1163 defer t.reqMu.Unlock() 1164 _, ok := t.reqCanceler[key] 1165 if !ok { 1166 return false 1167 } 1168 if fn != nil { 1169 t.reqCanceler[key] = fn 1170 } else { 1171 delete(t.reqCanceler, key) 1172 } 1173 return true 1174 } 1175 1176 var zeroDialer net.Dialer 1177 1178 func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { 1179 if t.DialContext != nil { 1180 return t.DialContext(ctx, network, addr) 1181 } 1182 if t.Dial != nil { 1183 c, err := t.Dial(network, addr) 1184 if c == nil && err == nil { 1185 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") 1186 } 1187 return c, err 1188 } 1189 return zeroDialer.DialContext(ctx, network, addr) 1190 } 1191 1192 // A wantConn records state about a wanted connection 1193 // (that is, an active call to getConn). 1194 // The conn may be gotten by dialing or by finding an idle connection, 1195 // or a cancellation may make the conn no longer wanted. 1196 // These three options are racing against each other and use 1197 // wantConn to coordinate and agree about the winning outcome. 1198 type wantConn struct { 1199 cm connectMethod 1200 key connectMethodKey // cm.Key() 1201 ctx context.Context // context for dial 1202 ready chan struct{} // closed when pc, err pair is delivered 1203 1204 // hooks for testing to know when dials are done 1205 // beforeDial is called in the getConn goroutine when the dial is queued. 1206 // afterDial is called when the dial is completed or canceled. 1207 beforeDial func() 1208 afterDial func() 1209 1210 mu sync.Mutex // protects pc, err, close(ready) 1211 pc *persistConn 1212 err error 1213 } 1214 1215 // waiting reports whether w is still waiting for an answer (connection or error). 1216 func (w *wantConn) waiting() bool { 1217 select { 1218 case <-w.ready: 1219 return false 1220 default: 1221 return true 1222 } 1223 } 1224 1225 // tryDeliver attempts to deliver pc, err to w and reports whether it succeeded. 1226 func (w *wantConn) tryDeliver(pc *persistConn, err error) bool { 1227 w.mu.Lock() 1228 defer w.mu.Unlock() 1229 1230 if w.pc != nil || w.err != nil { 1231 return false 1232 } 1233 1234 w.pc = pc 1235 w.err = err 1236 if w.pc == nil && w.err == nil { 1237 panic("net/http: internal error: misuse of tryDeliver") 1238 } 1239 close(w.ready) 1240 return true 1241 } 1242 1243 // cancel marks w as no longer wanting a result (for example, due to cancellation). 1244 // If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn. 1245 func (w *wantConn) cancel(t *Transport, err error) { 1246 w.mu.Lock() 1247 if w.pc == nil && w.err == nil { 1248 close(w.ready) // catch misbehavior in future delivery 1249 } 1250 pc := w.pc 1251 w.pc = nil 1252 w.err = err 1253 w.mu.Unlock() 1254 1255 if pc != nil { 1256 t.putOrCloseIdleConn(pc) 1257 } 1258 } 1259 1260 // A wantConnQueue is a queue of wantConns. 1261 type wantConnQueue struct { 1262 // This is a queue, not a deque. 1263 // It is split into two stages - head[headPos:] and tail. 1264 // popFront is trivial (headPos++) on the first stage, and 1265 // pushBack is trivial (append) on the second stage. 1266 // If the first stage is empty, popFront can swap the 1267 // first and second stages to remedy the situation. 1268 // 1269 // This two-stage split is analogous to the use of two lists 1270 // in Okasaki's purely functional queue but without the 1271 // overhead of reversing the list when swapping stages. 1272 head []*wantConn 1273 headPos int 1274 tail []*wantConn 1275 } 1276 1277 // len returns the number of items in the queue. 1278 func (q *wantConnQueue) len() int { 1279 return len(q.head) - q.headPos + len(q.tail) 1280 } 1281 1282 // pushBack adds w to the back of the queue. 1283 func (q *wantConnQueue) pushBack(w *wantConn) { 1284 q.tail = append(q.tail, w) 1285 } 1286 1287 // popFront removes and returns the wantConn at the front of the queue. 1288 func (q *wantConnQueue) popFront() *wantConn { 1289 if q.headPos >= len(q.head) { 1290 if len(q.tail) == 0 { 1291 return nil 1292 } 1293 // Pick up tail as new head, clear tail. 1294 q.head, q.headPos, q.tail = q.tail, 0, q.head[:0] 1295 } 1296 w := q.head[q.headPos] 1297 q.head[q.headPos] = nil 1298 q.headPos++ 1299 return w 1300 } 1301 1302 // peekFront returns the wantConn at the front of the queue without removing it. 1303 func (q *wantConnQueue) peekFront() *wantConn { 1304 if q.headPos < len(q.head) { 1305 return q.head[q.headPos] 1306 } 1307 if len(q.tail) > 0 { 1308 return q.tail[0] 1309 } 1310 return nil 1311 } 1312 1313 // cleanFront pops any wantConns that are no longer waiting from the head of the 1314 // queue, reporting whether any were popped. 1315 func (q *wantConnQueue) cleanFront() (cleaned bool) { 1316 for { 1317 w := q.peekFront() 1318 if w == nil || w.waiting() { 1319 return cleaned 1320 } 1321 q.popFront() 1322 cleaned = true 1323 } 1324 } 1325 1326 func (t *Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) { 1327 if t.DialTLSContext != nil { 1328 conn, err = t.DialTLSContext(ctx, network, addr) 1329 } else { 1330 conn, err = t.DialTLS(network, addr) 1331 } 1332 if conn == nil && err == nil { 1333 err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)") 1334 } 1335 return 1336 } 1337 1338 // getConn dials and creates a new persistConn to the target as 1339 // specified in the connectMethod. This includes doing a proxy CONNECT 1340 // and/or setting up TLS. If this doesn't return an error, the persistConn 1341 // is ready to write requests to. 1342 func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (pc *persistConn, err error) { 1343 req := treq.Request 1344 trace := treq.trace 1345 ctx := req.Context() 1346 if trace != nil && trace.GetConn != nil { 1347 trace.GetConn(cm.addr()) 1348 } 1349 1350 w := &wantConn{ 1351 cm: cm, 1352 key: cm.key(), 1353 ctx: ctx, 1354 ready: make(chan struct{}, 1), 1355 beforeDial: testHookPrePendingDial, 1356 afterDial: testHookPostPendingDial, 1357 } 1358 defer func() { 1359 if err != nil { 1360 w.cancel(t, err) 1361 } 1362 }() 1363 1364 // Queue for idle connection. 1365 if delivered := t.queueForIdleConn(w); delivered { 1366 pc := w.pc 1367 // Trace only for HTTP/1. 1368 // HTTP/2 calls trace.GotConn itself. 1369 if pc.alt == nil && trace != nil && trace.GotConn != nil { 1370 trace.GotConn(pc.gotIdleConnTrace(pc.idleAt)) 1371 } 1372 // set request canceler to some non-nil function so we 1373 // can detect whether it was cleared between now and when 1374 // we enter roundTrip 1375 t.setReqCanceler(treq.cancelKey, func(error) {}) 1376 return pc, nil 1377 } 1378 1379 cancelc := make(chan error, 1) 1380 t.setReqCanceler(treq.cancelKey, func(err error) { cancelc <- err }) 1381 1382 // Queue for permission to dial. 1383 t.queueForDial(w) 1384 1385 // Wait for completion or cancellation. 1386 select { 1387 case <-w.ready: 1388 // Trace success but only for HTTP/1. 1389 // HTTP/2 calls trace.GotConn itself. 1390 if w.pc != nil && w.pc.alt == nil && trace != nil && trace.GotConn != nil { 1391 trace.GotConn(httptrace.GotConnInfo{Conn: w.pc.conn, Reused: w.pc.isReused()}) 1392 } 1393 if w.err != nil { 1394 // If the request has been canceled, that's probably 1395 // what caused w.err; if so, prefer to return the 1396 // cancellation error (see golang.org/issue/16049). 1397 select { 1398 case <-req.Cancel: 1399 return nil, errRequestCanceledConn 1400 case <-req.Context().Done(): 1401 return nil, req.Context().Err() 1402 case err := <-cancelc: 1403 if err == errRequestCanceled { 1404 err = errRequestCanceledConn 1405 } 1406 return nil, err 1407 default: 1408 // return below 1409 } 1410 } 1411 return w.pc, w.err 1412 case <-req.Cancel: 1413 return nil, errRequestCanceledConn 1414 case <-req.Context().Done(): 1415 return nil, req.Context().Err() 1416 case err := <-cancelc: 1417 if err == errRequestCanceled { 1418 err = errRequestCanceledConn 1419 } 1420 return nil, err 1421 } 1422 } 1423 1424 // queueForDial queues w to wait for permission to begin dialing. 1425 // Once w receives permission to dial, it will do so in a separate goroutine. 1426 func (t *Transport) queueForDial(w *wantConn) { 1427 w.beforeDial() 1428 if t.MaxConnsPerHost <= 0 { 1429 go t.dialConnFor(w) 1430 return 1431 } 1432 1433 t.connsPerHostMu.Lock() 1434 defer t.connsPerHostMu.Unlock() 1435 1436 if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost { 1437 if t.connsPerHost == nil { 1438 t.connsPerHost = make(map[connectMethodKey]int) 1439 } 1440 t.connsPerHost[w.key] = n + 1 1441 go t.dialConnFor(w) 1442 return 1443 } 1444 1445 if t.connsPerHostWait == nil { 1446 t.connsPerHostWait = make(map[connectMethodKey]wantConnQueue) 1447 } 1448 q := t.connsPerHostWait[w.key] 1449 q.cleanFront() 1450 q.pushBack(w) 1451 t.connsPerHostWait[w.key] = q 1452 } 1453 1454 // dialConnFor dials on behalf of w and delivers the result to w. 1455 // dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.Key()]. 1456 // If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.Key()]. 1457 func (t *Transport) dialConnFor(w *wantConn) { 1458 defer w.afterDial() 1459 1460 pc, err := t.dialConn(w.ctx, w.cm) 1461 delivered := w.tryDeliver(pc, err) 1462 if err == nil && (!delivered || pc.alt != nil) { 1463 // pconn was not passed to w, 1464 // or it is HTTP/2 and can be shared. 1465 // Add to the idle connection pool. 1466 t.putOrCloseIdleConn(pc) 1467 } 1468 if err != nil { 1469 t.decConnsPerHost(w.key) 1470 } 1471 } 1472 1473 // decConnsPerHost decrements the per-host connection count for Key, 1474 // which may in turn give a different waiting goroutine permission to dial. 1475 func (t *Transport) decConnsPerHost(key connectMethodKey) { 1476 if t.MaxConnsPerHost <= 0 { 1477 return 1478 } 1479 1480 t.connsPerHostMu.Lock() 1481 defer t.connsPerHostMu.Unlock() 1482 n := t.connsPerHost[key] 1483 if n == 0 { 1484 // Shouldn't happen, but if it does, the counting is buggy and could 1485 // easily lead to a silent deadlock, so report the problem loudly. 1486 panic("net/http: internal error: connCount underflow") 1487 } 1488 1489 // Can we hand this count to a goroutine still waiting to dial? 1490 // (Some goroutines on the wait list may have timed out or 1491 // gotten a connection another way. If they're all gone, 1492 // we don't want to kick off any spurious dial operations.) 1493 if q := t.connsPerHostWait[key]; q.len() > 0 { 1494 done := false 1495 for q.len() > 0 { 1496 w := q.popFront() 1497 if w.waiting() { 1498 go t.dialConnFor(w) 1499 done = true 1500 break 1501 } 1502 } 1503 if q.len() == 0 { 1504 delete(t.connsPerHostWait, key) 1505 } else { 1506 // q is a value (like a slice), so we have to store 1507 // the updated q back into the map. 1508 t.connsPerHostWait[key] = q 1509 } 1510 if done { 1511 return 1512 } 1513 } 1514 1515 // Otherwise, decrement the recorded count. 1516 if n--; n == 0 { 1517 delete(t.connsPerHost, key) 1518 } else { 1519 t.connsPerHost[key] = n 1520 } 1521 } 1522 1523 // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS 1524 // tunnel, this function establishes a nested TLS session inside the encrypted channel. 1525 // The remote endpoint's name may be overridden by TLSClientConfig.ServerName. 1526 func (pconn *persistConn) addTLS(ctx context.Context, name string, trace *httptrace.ClientTrace) error { 1527 // Initiate TLS and check remote host name against certificate. 1528 cfg := cloneTLSConfig(pconn.t.TLSClientConfig) 1529 if cfg.ServerName == "" { 1530 cfg.ServerName = name 1531 } 1532 if pconn.cacheKey.onlyH1 { 1533 cfg.NextProtos = nil 1534 } 1535 plainConn := pconn.conn 1536 tlsConn := tls.Client(plainConn, cfg) 1537 errc := make(chan error, 2) 1538 var timer *time.Timer // for canceling TLS handshake 1539 if d := pconn.t.TLSHandshakeTimeout; d != 0 { 1540 timer = time.AfterFunc(d, func() { 1541 errc <- tlsHandshakeTimeoutError{} 1542 }) 1543 } 1544 go func() { 1545 if trace != nil && trace.TLSHandshakeStart != nil { 1546 trace.TLSHandshakeStart() 1547 } 1548 err := tlsConn.HandshakeContext(ctx) 1549 if timer != nil { 1550 timer.Stop() 1551 } 1552 errc <- err 1553 }() 1554 if err := <-errc; err != nil { 1555 plainConn.Close() 1556 if trace != nil && trace.TLSHandshakeDone != nil { 1557 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1558 } 1559 return err 1560 } 1561 cs := tlsConn.ConnectionState() 1562 if trace != nil && trace.TLSHandshakeDone != nil { 1563 trace.TLSHandshakeDone(cs, nil) 1564 } 1565 pconn.tlsState = &cs 1566 pconn.conn = tlsConn 1567 return nil 1568 } 1569 1570 type erringRoundTripper interface { 1571 RoundTripErr() error 1572 } 1573 1574 func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *persistConn, err error) { 1575 pconn = &persistConn{ 1576 t: t, 1577 cacheKey: cm.key(), 1578 reqch: make(chan requestAndChan, 1), 1579 writech: make(chan writeRequest, 1), 1580 closech: make(chan struct{}), 1581 writeErrCh: make(chan error, 1), 1582 writeLoopDone: make(chan struct{}), 1583 } 1584 trace := httptrace.ContextClientTrace(ctx) 1585 wrapErr := func(err error) error { 1586 if cm.proxyURL != nil { 1587 // Return a typed error, per Issue 16997 1588 return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} 1589 } 1590 return err 1591 } 1592 if cm.scheme() == "https" && t.hasCustomTLSDialer() { 1593 var err error 1594 pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr()) 1595 if err != nil { 1596 return nil, wrapErr(err) 1597 } 1598 if tc, ok := pconn.conn.(*tls.Conn); ok { 1599 // Handshake here, in case DialTLS didn't. TLSNextProto below 1600 // depends on it for knowing the connection state. 1601 if trace != nil && trace.TLSHandshakeStart != nil { 1602 trace.TLSHandshakeStart() 1603 } 1604 if err := tc.HandshakeContext(ctx); err != nil { 1605 go pconn.conn.Close() 1606 if trace != nil && trace.TLSHandshakeDone != nil { 1607 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1608 } 1609 return nil, err 1610 } 1611 cs := tc.ConnectionState() 1612 if trace != nil && trace.TLSHandshakeDone != nil { 1613 trace.TLSHandshakeDone(cs, nil) 1614 } 1615 pconn.tlsState = &cs 1616 } 1617 } else { 1618 conn, err := t.dial(ctx, "tcp", cm.addr()) 1619 if err != nil { 1620 return nil, wrapErr(err) 1621 } 1622 pconn.conn = conn 1623 if cm.scheme() == "https" { 1624 var firstTLSHost string 1625 if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { 1626 return nil, wrapErr(err) 1627 } 1628 if err = pconn.addTLS(ctx, firstTLSHost, trace); err != nil { 1629 return nil, wrapErr(err) 1630 } 1631 } 1632 } 1633 1634 // Proxy setup. 1635 switch { 1636 case cm.proxyURL == nil: 1637 // Do nothing. Not using a proxy. 1638 case cm.proxyURL.Scheme == "socks5": 1639 conn := pconn.conn 1640 d := socksNewDialer("tcp", conn.RemoteAddr().String()) 1641 if u := cm.proxyURL.User; u != nil { 1642 auth := &socksUsernamePassword{ 1643 Username: u.Username(), 1644 } 1645 auth.Password, _ = u.Password() 1646 d.AuthMethods = []socksAuthMethod{ 1647 socksAuthMethodNotRequired, 1648 socksAuthMethodUsernamePassword, 1649 } 1650 d.Authenticate = auth.Authenticate 1651 } 1652 if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { 1653 conn.Close() 1654 return nil, err 1655 } 1656 case cm.targetScheme == "http": 1657 pconn.isProxy = true 1658 if pa := cm.proxyAuth(); pa != "" { 1659 pconn.mutateHeaderFunc = func(h Header) { 1660 h.Set("Proxy-Authorization", pa) 1661 } 1662 } 1663 case cm.targetScheme == "https": 1664 conn := pconn.conn 1665 var hdr Header 1666 if t.GetProxyConnectHeader != nil { 1667 var err error 1668 hdr, err = t.GetProxyConnectHeader(ctx, cm.proxyURL, cm.targetAddr) 1669 if err != nil { 1670 conn.Close() 1671 return nil, err 1672 } 1673 } else { 1674 hdr = t.ProxyConnectHeader 1675 } 1676 if hdr == nil { 1677 hdr = make(Header) 1678 } 1679 if pa := cm.proxyAuth(); pa != "" { 1680 hdr = hdr.Clone() 1681 hdr.Set("Proxy-Authorization", pa) 1682 } 1683 connectReq := &Request{ 1684 Method: "CONNECT", 1685 URL: &url.URL{Opaque: cm.targetAddr}, 1686 Host: cm.targetAddr, 1687 Header: hdr, 1688 } 1689 1690 // If there's no done channel (no deadline or cancellation 1691 // from the caller possible), at least set some (long) 1692 // timeout here. This will make sure we don't block forever 1693 // and leak a goroutine if the connection stops replying 1694 // after the TCP connect. 1695 connectCtx := ctx 1696 if ctx.Done() == nil { 1697 newCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) 1698 defer cancel() 1699 connectCtx = newCtx 1700 } 1701 1702 didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails 1703 var ( 1704 resp *Response 1705 err error // write or read error 1706 ) 1707 // Write the CONNECT request & read the response. 1708 go func() { 1709 defer close(didReadResponse) 1710 err = connectReq.Write(conn) 1711 if err != nil { 1712 return 1713 } 1714 // Okay to use and discard buffered reader here, because 1715 // TLS server will not speak until spoken to. 1716 br := bufio.NewReader(conn) 1717 resp, err = ReadResponse(br, connectReq) 1718 }() 1719 select { 1720 case <-connectCtx.Done(): 1721 conn.Close() 1722 <-didReadResponse 1723 return nil, connectCtx.Err() 1724 case <-didReadResponse: 1725 // resp or err now set 1726 } 1727 if err != nil { 1728 conn.Close() 1729 return nil, err 1730 } 1731 1732 if t.OnProxyConnectResponse != nil { 1733 err = t.OnProxyConnectResponse(ctx, cm.proxyURL, connectReq, resp) 1734 if err != nil { 1735 return nil, err 1736 } 1737 } 1738 1739 if resp.StatusCode != 200 { 1740 _, text, ok := strings.Cut(resp.Status, " ") 1741 conn.Close() 1742 if !ok { 1743 return nil, errors.New("unknown status code") 1744 } 1745 return nil, errors.New(text) 1746 } 1747 } 1748 1749 if cm.proxyURL != nil && cm.targetScheme == "https" { 1750 if err := pconn.addTLS(ctx, cm.tlsHost(), trace); err != nil { 1751 return nil, err 1752 } 1753 } 1754 1755 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 1756 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 1757 alt := next(cm.targetAddr, pconn.conn.(*tls.Conn)) 1758 if e, ok := alt.(erringRoundTripper); ok { 1759 // pconn.conn was closed by next (http2configureTransports.upgradeFn). 1760 return nil, e.RoundTripErr() 1761 } 1762 return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil 1763 } 1764 } 1765 1766 pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize()) 1767 pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize()) 1768 1769 go pconn.readLoop() 1770 go pconn.writeLoop() 1771 return pconn, nil 1772 } 1773 1774 // persistConnWriter is the io.Writer written to by pc.bw. 1775 // It accumulates the number of bytes written to the underlying conn, 1776 // so the retry logic can determine whether any bytes made it across 1777 // the wire. 1778 // This is exactly 1 pointer field wide so it can go into an interface 1779 // without allocation. 1780 type persistConnWriter struct { 1781 pc *persistConn 1782 } 1783 1784 func (w persistConnWriter) Write(p []byte) (n int, err error) { 1785 n, err = w.pc.conn.Write(p) 1786 w.pc.nwrite += int64(n) 1787 return 1788 } 1789 1790 // ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if 1791 // the Conn implements io.ReaderFrom, it can take advantage of optimizations 1792 // such as sendfile. 1793 func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) { 1794 n, err = io.Copy(w.pc.conn, r) 1795 w.pc.nwrite += n 1796 return 1797 } 1798 1799 var _ io.ReaderFrom = (*persistConnWriter)(nil) 1800 1801 // connectMethod is the map key (in its String form) for keeping persistent 1802 // TCP connections alive for subsequent HTTP requests. 1803 // 1804 // A connect method may be of the following types: 1805 // 1806 // connectMethod.Key().String() Description 1807 // ------------------------------ ------------------------- 1808 // |http|foo.com http directly to server, no proxy 1809 // |https|foo.com https directly to server, no proxy 1810 // |https,h1|foo.com https directly to server w/o HTTP/2, no proxy 1811 // http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 1812 // http://proxy.com|http http to proxy, http to anywhere after that 1813 // socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com 1814 // socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com 1815 // https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com 1816 // https://proxy.com|http https to proxy, http to anywhere after that 1817 type connectMethod struct { 1818 _ incomparable 1819 proxyURL *url.URL // nil for no proxy, else full proxy URL 1820 targetScheme string // "http" or "https" 1821 // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), 1822 // then targetAddr is not included in the connect method key, because the socket can 1823 // be reused for different targetAddr Values. 1824 targetAddr string 1825 onlyH1 bool // whether to disable HTTP/2 and force HTTP/1 1826 } 1827 1828 func (cm *connectMethod) key() connectMethodKey { 1829 proxyStr := "" 1830 targetAddr := cm.targetAddr 1831 if cm.proxyURL != nil { 1832 proxyStr = cm.proxyURL.String() 1833 if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { 1834 targetAddr = "" 1835 } 1836 } 1837 return connectMethodKey{ 1838 proxy: proxyStr, 1839 scheme: cm.targetScheme, 1840 addr: targetAddr, 1841 onlyH1: cm.onlyH1, 1842 } 1843 } 1844 1845 // scheme returns the first hop scheme: http, https, or socks5 1846 func (cm *connectMethod) scheme() string { 1847 if cm.proxyURL != nil { 1848 return cm.proxyURL.Scheme 1849 } 1850 return cm.targetScheme 1851 } 1852 1853 // addr returns the first hop "host:port" to which we need to TCP connect. 1854 func (cm *connectMethod) addr() string { 1855 if cm.proxyURL != nil { 1856 return canonicalAddr(cm.proxyURL) 1857 } 1858 return cm.targetAddr 1859 } 1860 1861 // tlsHost returns the host name to match against the peer's 1862 // TLS certificate. 1863 func (cm *connectMethod) tlsHost() string { 1864 h := cm.targetAddr 1865 if hasPort(h) { 1866 h = h[:strings.LastIndex(h, ":")] 1867 } 1868 return h 1869 } 1870 1871 // connectMethodKey is the map Key version of connectMethod, with a 1872 // stringified proxy URL (or the empty string) instead of a pointer to 1873 // a URL. 1874 type connectMethodKey struct { 1875 proxy, scheme, addr string 1876 onlyH1 bool 1877 } 1878 1879 func (k connectMethodKey) String() string { 1880 // Only used by tests. 1881 var h1 string 1882 if k.onlyH1 { 1883 h1 = ",h1" 1884 } 1885 return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr) 1886 } 1887 1888 // persistConn wraps a connection, usually a persistent one 1889 // (but may be used for non-keep-alive requests as well) 1890 type persistConn struct { 1891 // alt optionally specifies the TLS NextProto RoundTripper. 1892 // This is used for HTTP/2 today and future protocols later. 1893 // If it's non-nil, the rest of the fields are unused. 1894 alt RoundTripper 1895 1896 t *Transport 1897 cacheKey connectMethodKey 1898 conn net.Conn 1899 tlsState *tls.ConnectionState 1900 br *bufio.Reader // from conn 1901 bw *bufio.Writer // to conn 1902 nwrite int64 // bytes written 1903 reqch chan requestAndChan // written by roundTrip; read by readLoop 1904 writech chan writeRequest // written by roundTrip; read by writeLoop 1905 closech chan struct{} // closed when conn closed 1906 isProxy bool 1907 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 1908 readLimit int64 // bytes allowed to be read; owned by readLoop 1909 // writeErrCh passes the request write error (usually nil) 1910 // from the writeLoop goroutine to the readLoop which passes 1911 // it off to the res.Body reader, which then uses it to decide 1912 // whether or not a connection can be reused. Issue 7569. 1913 writeErrCh chan error 1914 1915 writeLoopDone chan struct{} // closed when write loop ends 1916 1917 // Both guarded by Transport.idleMu: 1918 idleAt time.Time // time it last become idle 1919 idleTimer *time.Timer // holding an AfterFunc to close it 1920 1921 mu sync.Mutex // guards following fields 1922 numExpectedResponses int 1923 closed error // set non-nil when conn is closed, before closech is closed 1924 canceledErr error // set non-nil if conn is canceled 1925 broken bool // an error has happened on this connection; marked broken so it's not reused. 1926 reused bool // whether conn has had successful request/response and is being reused. 1927 // mutateHeaderFunc is an optional func to modify extra 1928 // headers on each outbound request before it's written. (the 1929 // original Request given to RoundTrip is not modified) 1930 mutateHeaderFunc func(Header) 1931 } 1932 1933 func (pc *persistConn) maxHeaderResponseSize() int64 { 1934 if v := pc.t.MaxResponseHeaderBytes; v != 0 { 1935 return v 1936 } 1937 return 10 << 20 // conservative default; same as http2 1938 } 1939 1940 func (pc *persistConn) Read(p []byte) (n int, err error) { 1941 if pc.readLimit <= 0 { 1942 return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) 1943 } 1944 if int64(len(p)) > pc.readLimit { 1945 p = p[:pc.readLimit] 1946 } 1947 n, err = pc.conn.Read(p) 1948 if err == io.EOF { 1949 pc.sawEOF = true 1950 } 1951 pc.readLimit -= int64(n) 1952 return 1953 } 1954 1955 // isBroken reports whether this connection is in a known broken state. 1956 func (pc *persistConn) isBroken() bool { 1957 pc.mu.Lock() 1958 b := pc.closed != nil 1959 pc.mu.Unlock() 1960 return b 1961 } 1962 1963 // canceled returns non-nil if the connection was closed due to 1964 // CancelRequest or due to context cancellation. 1965 func (pc *persistConn) canceled() error { 1966 pc.mu.Lock() 1967 defer pc.mu.Unlock() 1968 return pc.canceledErr 1969 } 1970 1971 // isReused reports whether this connection has been used before. 1972 func (pc *persistConn) isReused() bool { 1973 pc.mu.Lock() 1974 r := pc.reused 1975 pc.mu.Unlock() 1976 return r 1977 } 1978 1979 func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { 1980 pc.mu.Lock() 1981 defer pc.mu.Unlock() 1982 t.Reused = pc.reused 1983 t.Conn = pc.conn 1984 t.WasIdle = true 1985 if !idleAt.IsZero() { 1986 t.IdleTime = time.Since(idleAt) 1987 } 1988 return 1989 } 1990 1991 func (pc *persistConn) cancelRequest(err error) { 1992 pc.mu.Lock() 1993 defer pc.mu.Unlock() 1994 pc.canceledErr = err 1995 pc.closeLocked(errRequestCanceled) 1996 } 1997 1998 // closeConnIfStillIdle closes the connection if it's still sitting idle. 1999 // This is what's called by the persistConn's idleTimer, and is run in its 2000 // own goroutine. 2001 func (pc *persistConn) closeConnIfStillIdle() { 2002 t := pc.t 2003 t.idleMu.Lock() 2004 defer t.idleMu.Unlock() 2005 if _, ok := t.idleLRU.m[pc]; !ok { 2006 // Not idle. 2007 return 2008 } 2009 t.removeIdleConnLocked(pc) 2010 pc.close(errIdleConnTimeout) 2011 } 2012 2013 // mapRoundTripError returns the appropriate error value for 2014 // persistConn.roundTrip. 2015 // 2016 // The provided err is the first error that (*persistConn).roundTrip 2017 // happened to receive from its select statement. 2018 // 2019 // The startBytesWritten value should be the value of pc.nwrite before the roundTrip 2020 // started writing the request. 2021 func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { 2022 if err == nil { 2023 return nil 2024 } 2025 2026 // Wait for the writeLoop goroutine to terminate to avoid data 2027 // races on callers who mutate the request on failure. 2028 // 2029 // When resc in pc.roundTrip and hence rc.ch receives a responseAndError 2030 // with a non-nil error it implies that the persistConn is either closed 2031 // or closing. Waiting on pc.writeLoopDone is hence safe as all callers 2032 // close closech which in turn ensures writeLoop returns. 2033 <-pc.writeLoopDone 2034 2035 // If the request was canceled, that's better than network 2036 // failures that were likely the result of tearing down the 2037 // connection. 2038 if cerr := pc.canceled(); cerr != nil { 2039 return cerr 2040 } 2041 2042 // See if an error was set explicitly. 2043 req.mu.Lock() 2044 reqErr := req.err 2045 req.mu.Unlock() 2046 if reqErr != nil { 2047 return reqErr 2048 } 2049 2050 if err == errServerClosedIdle { 2051 // Don't decorate 2052 return err 2053 } 2054 2055 if _, ok := err.(transportReadFromServerError); ok { 2056 if pc.nwrite == startBytesWritten { 2057 return nothingWrittenError{err} 2058 } 2059 // Don't decorate 2060 return err 2061 } 2062 if pc.isBroken() { 2063 if pc.nwrite == startBytesWritten { 2064 return nothingWrittenError{err} 2065 } 2066 return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %w", err) 2067 } 2068 return err 2069 } 2070 2071 // errCallerOwnsConn is an internal sentinel error used when we hand 2072 // off a writable response.Body to the caller. We use this to prevent 2073 // closing a net.Conn that is now owned by the caller. 2074 var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn") 2075 2076 func (pc *persistConn) readLoop() { 2077 closeErr := errReadLoopExiting // default value, if not changed below 2078 defer func() { 2079 pc.close(closeErr) 2080 pc.t.removeIdleConn(pc) 2081 }() 2082 2083 tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { 2084 if err := pc.t.tryPutIdleConn(pc); err != nil { 2085 closeErr = err 2086 if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { 2087 trace.PutIdleConn(err) 2088 } 2089 return false 2090 } 2091 if trace != nil && trace.PutIdleConn != nil { 2092 trace.PutIdleConn(nil) 2093 } 2094 return true 2095 } 2096 2097 // eofc is used to block caller goroutines reading from Response.Body 2098 // at EOF until this goroutines has (potentially) added the connection 2099 // back to the idle pool. 2100 eofc := make(chan struct{}) 2101 defer close(eofc) // unblock reader on errors 2102 2103 // Read this once, before loop starts. (to avoid races in tests) 2104 testHookMu.Lock() 2105 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 2106 testHookMu.Unlock() 2107 2108 alive := true 2109 for alive { 2110 pc.readLimit = pc.maxHeaderResponseSize() 2111 _, err := pc.br.Peek(1) 2112 2113 pc.mu.Lock() 2114 if pc.numExpectedResponses == 0 { 2115 pc.readLoopPeekFailLocked(err) 2116 pc.mu.Unlock() 2117 return 2118 } 2119 pc.mu.Unlock() 2120 2121 rc := <-pc.reqch 2122 trace := httptrace.ContextClientTrace(rc.req.Context()) 2123 2124 var resp *Response 2125 if err == nil { 2126 resp, err = pc.readResponse(rc, trace) 2127 } else { 2128 err = transportReadFromServerError{err} 2129 closeErr = err 2130 } 2131 2132 if err != nil { 2133 if pc.readLimit <= 0 { 2134 err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) 2135 } 2136 2137 select { 2138 case rc.ch <- responseAndError{err: err}: 2139 case <-rc.callerGone: 2140 return 2141 } 2142 return 2143 } 2144 pc.readLimit = maxInt64 // effectively no limit for response bodies 2145 2146 pc.mu.Lock() 2147 pc.numExpectedResponses-- 2148 pc.mu.Unlock() 2149 2150 bodyWritable := resp.bodyIsWritable() 2151 hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 2152 2153 if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable { 2154 // Don't do keep-alive on error if either party requested a close 2155 // or we get an unexpected informational (1xx) response. 2156 // StatusCode 100 is already handled above. 2157 alive = false 2158 } 2159 2160 if !hasBody || bodyWritable { 2161 replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) 2162 2163 // Put the idle conn back into the pool before we send the response 2164 // so if they process it quickly and make another request, they'll 2165 // get this same conn. But we use the unbuffered channel 'rc' 2166 // to guarantee that persistConn.roundTrip got out of its select 2167 // potentially waiting for this persistConn to close. 2168 alive = alive && 2169 !pc.sawEOF && 2170 pc.wroteRequest() && 2171 replaced && tryPutIdleConn(trace) 2172 2173 if bodyWritable { 2174 closeErr = errCallerOwnsConn 2175 } 2176 2177 select { 2178 case rc.ch <- responseAndError{res: resp}: 2179 case <-rc.callerGone: 2180 return 2181 } 2182 2183 // Now that they've read from the unbuffered channel, they're safely 2184 // out of the select that also waits on this goroutine to die, so 2185 // we're allowed to exit now if needed (if alive is false) 2186 testHookReadLoopBeforeNextRead() 2187 continue 2188 } 2189 2190 waitForBodyRead := make(chan bool, 2) 2191 body := &bodyEOFSignal{ 2192 body: resp.Body, 2193 earlyCloseFn: func() error { 2194 waitForBodyRead <- false 2195 <-eofc // will be closed by deferred call at the end of the function 2196 return nil 2197 2198 }, 2199 fn: func(err error) error { 2200 isEOF := err == io.EOF 2201 waitForBodyRead <- isEOF 2202 if isEOF { 2203 <-eofc // see comment above eofc declaration 2204 } else if err != nil { 2205 if cerr := pc.canceled(); cerr != nil { 2206 return cerr 2207 } 2208 } 2209 return err 2210 }, 2211 } 2212 2213 resp.Body = body 2214 if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { 2215 resp.Body = &gzipReader{body: body} 2216 resp.Header.Del("Content-Encoding") 2217 resp.Header.Del("Content-Length") 2218 resp.ContentLength = -1 2219 resp.Uncompressed = true 2220 } 2221 2222 select { 2223 case rc.ch <- responseAndError{res: resp}: 2224 case <-rc.callerGone: 2225 return 2226 } 2227 2228 // Before looping back to the top of this function and peeking on 2229 // the bufio.Reader, wait for the caller goroutine to finish 2230 // reading the response body. (or for cancellation or death) 2231 select { 2232 case bodyEOF := <-waitForBodyRead: 2233 replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool 2234 alive = alive && 2235 bodyEOF && 2236 !pc.sawEOF && 2237 pc.wroteRequest() && 2238 replaced && tryPutIdleConn(trace) 2239 if bodyEOF { 2240 eofc <- struct{}{} 2241 } 2242 case <-rc.req.Cancel: 2243 alive = false 2244 pc.t.CancelRequest(rc.req) 2245 case <-rc.req.Context().Done(): 2246 alive = false 2247 pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) 2248 case <-pc.closech: 2249 alive = false 2250 } 2251 2252 testHookReadLoopBeforeNextRead() 2253 } 2254 } 2255 2256 func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 2257 if pc.closed != nil { 2258 return 2259 } 2260 if n := pc.br.Buffered(); n > 0 { 2261 buf, _ := pc.br.Peek(n) 2262 if is408Message(buf) { 2263 pc.closeLocked(errServerClosedIdle) 2264 return 2265 } else { 2266 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 2267 } 2268 } 2269 if peekErr == io.EOF { 2270 // common case. 2271 pc.closeLocked(errServerClosedIdle) 2272 } else { 2273 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %w", peekErr)) 2274 } 2275 } 2276 2277 // is408Message reports whether buf has the prefix of an 2278 // HTTP 408 Request Timeout response. 2279 // See golang.org/issue/32310. 2280 func is408Message(buf []byte) bool { 2281 if len(buf) < len("HTTP/1.x 408") { 2282 return false 2283 } 2284 if string(buf[:7]) != "HTTP/1." { 2285 return false 2286 } 2287 return string(buf[8:12]) == " 408" 2288 } 2289 2290 // readResponse reads an HTTP response (or two, in the case of "Expect: 2291 // 100-continue") from the server. It returns the final non-100 one. 2292 // trace is optional. 2293 func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { 2294 if trace != nil && trace.GotFirstResponseByte != nil { 2295 if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { 2296 trace.GotFirstResponseByte() 2297 } 2298 } 2299 num1xx := 0 // number of informational 1xx headers received 2300 const max1xxResponses = 5 // arbitrary bound on number of informational responses 2301 2302 continueCh := rc.continueCh 2303 for { 2304 resp, err = ReadResponse(pc.br, rc.req) 2305 if err != nil { 2306 return 2307 } 2308 resCode := resp.StatusCode 2309 if continueCh != nil { 2310 if resCode == 100 { 2311 if trace != nil && trace.Got100Continue != nil { 2312 trace.Got100Continue() 2313 } 2314 continueCh <- struct{}{} 2315 continueCh = nil 2316 } else if resCode >= 200 { 2317 close(continueCh) 2318 continueCh = nil 2319 } 2320 } 2321 is1xx := 100 <= resCode && resCode <= 199 2322 // treat 101 as a terminal status, see issue 26161 2323 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols 2324 if is1xxNonTerminal { 2325 num1xx++ 2326 if num1xx > max1xxResponses { 2327 return nil, errors.New("net/http: too many 1xx informational responses") 2328 } 2329 pc.readLimit = pc.maxHeaderResponseSize() // reset the limit 2330 if trace != nil && trace.Got1xxResponse != nil { 2331 if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { 2332 return nil, err 2333 } 2334 } 2335 continue 2336 } 2337 break 2338 } 2339 if resp.isProtocolSwitch() { 2340 resp.Body = newReadWriteCloserBody(pc.br, pc.conn) 2341 } 2342 2343 resp.TLS = pc.tlsState 2344 return 2345 } 2346 2347 // waitForContinue returns the function to block until 2348 // any response, timeout or connection close. After any of them, 2349 // the function returns a bool which indicates if the body should be sent. 2350 func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 2351 if continueCh == nil { 2352 return nil 2353 } 2354 return func() bool { 2355 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 2356 defer timer.Stop() 2357 2358 select { 2359 case _, ok := <-continueCh: 2360 return ok 2361 case <-timer.C: 2362 return true 2363 case <-pc.closech: 2364 return false 2365 } 2366 } 2367 } 2368 2369 func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser { 2370 body := &readWriteCloserBody{ReadWriteCloser: rwc} 2371 if br.Buffered() != 0 { 2372 body.br = br 2373 } 2374 return body 2375 } 2376 2377 // readWriteCloserBody is the Response.Body type used when we want to 2378 // give users write access to the Body through the underlying 2379 // connection (TCP, unless using custom dialers). This is then 2380 // the concrete type for a Response.Body on the 101 Switching 2381 // Protocols response, as used by WebSockets, h2c, etc. 2382 type readWriteCloserBody struct { 2383 _ incomparable 2384 br *bufio.Reader // used until empty 2385 io.ReadWriteCloser 2386 } 2387 2388 func (b *readWriteCloserBody) Read(p []byte) (n int, err error) { 2389 if b.br != nil { 2390 if n := b.br.Buffered(); len(p) > n { 2391 p = p[:n] 2392 } 2393 n, err = b.br.Read(p) 2394 if b.br.Buffered() == 0 { 2395 b.br = nil 2396 } 2397 return n, err 2398 } 2399 return b.ReadWriteCloser.Read(p) 2400 } 2401 2402 // nothingWrittenError wraps a write errors which ended up writing zero bytes. 2403 type nothingWrittenError struct { 2404 error 2405 } 2406 2407 func (nwe nothingWrittenError) Unwrap() error { 2408 return nwe.error 2409 } 2410 2411 func (pc *persistConn) writeLoop() { 2412 defer close(pc.writeLoopDone) 2413 for { 2414 select { 2415 case wr := <-pc.writech: 2416 startBytesWritten := pc.nwrite 2417 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 2418 if bre, ok := err.(requestBodyReadError); ok { 2419 err = bre.error 2420 // Errors reading from the user's 2421 // Request.Body are high priority. 2422 // Set it here before sending on the 2423 // channels below or calling 2424 // pc.close() which tears down 2425 // connections and causes other 2426 // errors. 2427 wr.req.setError(err) 2428 } 2429 if err == nil { 2430 err = pc.bw.Flush() 2431 } 2432 if err != nil { 2433 if pc.nwrite == startBytesWritten { 2434 err = nothingWrittenError{err} 2435 } 2436 } 2437 pc.writeErrCh <- err // to the body reader, which might recycle us 2438 wr.ch <- err // to the roundTrip function 2439 if err != nil { 2440 pc.close(err) 2441 return 2442 } 2443 case <-pc.closech: 2444 return 2445 } 2446 } 2447 } 2448 2449 // maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip 2450 // will wait to see the Request's Body.Write result after getting a 2451 // response from the server. See comments in (*persistConn).wroteRequest. 2452 const maxWriteWaitBeforeConnReuse = 50 * time.Millisecond 2453 2454 // wroteRequest is a check before recycling a connection that the previous write 2455 // (from writeLoop above) happened and was successful. 2456 func (pc *persistConn) wroteRequest() bool { 2457 select { 2458 case err := <-pc.writeErrCh: 2459 // Common case: the write happened well before the response, so 2460 // avoid creating a timer. 2461 return err == nil 2462 default: 2463 // Rare case: the request was written in writeLoop above but 2464 // before it could send to pc.writeErrCh, the reader read it 2465 // all, processed it, and called us here. In this case, give the 2466 // write goroutine a bit of time to finish its send. 2467 // 2468 // Less rare case: We also get here in the legitimate case of 2469 // Issue 7569, where the writer is still writing (or stalled), 2470 // but the server has already replied. In this case, we don't 2471 // want to wait too long, and we want to return false so this 2472 // connection isn't re-used. 2473 t := time.NewTimer(maxWriteWaitBeforeConnReuse) 2474 defer t.Stop() 2475 select { 2476 case err := <-pc.writeErrCh: 2477 return err == nil 2478 case <-t.C: 2479 return false 2480 } 2481 } 2482 } 2483 2484 // responseAndError is how the goroutine reading from an HTTP/1 server 2485 // communicates with the goroutine doing the RoundTrip. 2486 type responseAndError struct { 2487 _ incomparable 2488 res *Response // else use this response (see res method) 2489 err error 2490 } 2491 2492 type requestAndChan struct { 2493 _ incomparable 2494 req *Request 2495 cancelKey cancelKey 2496 ch chan responseAndError // unbuffered; always send in select on callerGone 2497 2498 // whether the Transport (as opposed to the user client code) 2499 // added the Accept-Encoding gzip header. If the Transport 2500 // set it, only then do we transparently decode the gzip. 2501 addedGzip bool 2502 2503 // Optional blocking chan for Expect: 100-continue (for send). 2504 // If the request has an "Expect: 100-continue" header and 2505 // the server responds 100 Continue, readLoop send a value 2506 // to writeLoop via this chan. 2507 continueCh chan<- struct{} 2508 2509 callerGone <-chan struct{} // closed when roundTrip caller has returned 2510 } 2511 2512 // A writeRequest is sent by the caller's goroutine to the 2513 // writeLoop's goroutine to write a request while the read loop 2514 // concurrently waits on both the write response and the server's 2515 // reply. 2516 type writeRequest struct { 2517 req *transportRequest 2518 ch chan<- error 2519 2520 // Optional blocking chan for Expect: 100-continue (for receive). 2521 // If not nil, writeLoop blocks sending request body until 2522 // it receives from this chan. 2523 continueCh <-chan struct{} 2524 } 2525 2526 type httpError struct { 2527 err string 2528 timeout bool 2529 } 2530 2531 func (e *httpError) Error() string { return e.err } 2532 func (e *httpError) Timeout() bool { return e.timeout } 2533 func (e *httpError) Temporary() bool { return true } 2534 2535 var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} 2536 2537 // errRequestCanceled is set to be identical to the one from h2 to facilitate 2538 // testing. 2539 var errRequestCanceled = http2errRequestCanceled 2540 var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? 2541 2542 func nop() {} 2543 2544 // testHooks. Always non-nil. 2545 var ( 2546 testHookEnterRoundTrip = nop 2547 testHookWaitResLoop = nop 2548 testHookRoundTripRetried = nop 2549 testHookPrePendingDial = nop 2550 testHookPostPendingDial = nop 2551 2552 testHookMu sync.Locker = fakeLocker{} // guards following 2553 testHookReadLoopBeforeNextRead = nop 2554 ) 2555 2556 func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 2557 testHookEnterRoundTrip() 2558 if !pc.t.replaceReqCanceler(req.cancelKey, pc.cancelRequest) { 2559 pc.t.putOrCloseIdleConn(pc) 2560 return nil, errRequestCanceled 2561 } 2562 pc.mu.Lock() 2563 pc.numExpectedResponses++ 2564 headerFn := pc.mutateHeaderFunc 2565 pc.mu.Unlock() 2566 2567 if headerFn != nil { 2568 headerFn(req.extraHeaders()) 2569 } 2570 2571 // Ask for a compressed version if the caller didn't set their 2572 // own value for Accept-Encoding. We only attempt to 2573 // uncompress the gzip stream if we were the layer that 2574 // requested it. 2575 requestedGzip := false 2576 if !pc.t.DisableCompression && 2577 req.Header.Get("Accept-Encoding") == "" && 2578 req.Header.Get("Range") == "" && 2579 req.Method != "HEAD" && 2580 strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { 2581 // Request gzip only, not deflate. Deflate is ambiguous and 2582 // not as universally supported anyway. 2583 // See: https://zlib.net/zlib_faq.html#faq39 2584 // 2585 // Note that we don't request this for HEAD requests, 2586 // due to a bug in nginx: 2587 // https://trac.nginx.org/nginx/ticket/358 2588 // https://golang.org/issue/5522 2589 // 2590 // We don't request gzip if the request is for a range, since 2591 // auto-decoding a portion of a gzipped document will just fail 2592 // anyway. See https://golang.org/issue/8923 2593 requestedGzip = true 2594 } 2595 2596 var continueCh chan struct{} 2597 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 2598 continueCh = make(chan struct{}, 1) 2599 } 2600 2601 if pc.t.DisableKeepAlives && 2602 !req.wantsClose() && 2603 !isProtocolSwitchHeader(req.Header) { 2604 req.extraHeaders().Set("Connection", "close") 2605 } 2606 2607 gone := make(chan struct{}) 2608 defer close(gone) 2609 2610 defer func() { 2611 if err != nil { 2612 pc.t.setReqCanceler(req.cancelKey, nil) 2613 } 2614 }() 2615 2616 const debugRoundTrip = false 2617 2618 // Write the request concurrently with waiting for a response, 2619 // in case the server decides to reply before reading our full 2620 // request body. 2621 startBytesWritten := pc.nwrite 2622 writeErrCh := make(chan error, 1) 2623 pc.writech <- writeRequest{req, writeErrCh, continueCh} 2624 2625 resc := make(chan responseAndError) 2626 pc.reqch <- requestAndChan{ 2627 req: req.Request, 2628 cancelKey: req.cancelKey, 2629 ch: resc, 2630 addedGzip: requestedGzip, 2631 continueCh: continueCh, 2632 callerGone: gone, 2633 } 2634 2635 var respHeaderTimer <-chan time.Time 2636 cancelChan := req.Request.Cancel 2637 ctxDoneChan := req.Context().Done() 2638 pcClosed := pc.closech 2639 canceled := false 2640 for { 2641 testHookWaitResLoop() 2642 select { 2643 case err := <-writeErrCh: 2644 if debugRoundTrip { 2645 req.logf("writeErrCh resv: %T/%#v", err, err) 2646 } 2647 if err != nil { 2648 pc.close(fmt.Errorf("write error: %w", err)) 2649 return nil, pc.mapRoundTripError(req, startBytesWritten, err) 2650 } 2651 if d := pc.t.ResponseHeaderTimeout; d > 0 { 2652 if debugRoundTrip { 2653 req.logf("starting timer for %v", d) 2654 } 2655 timer := time.NewTimer(d) 2656 defer timer.Stop() // prevent leaks 2657 respHeaderTimer = timer.C 2658 } 2659 case <-pcClosed: 2660 pcClosed = nil 2661 if canceled || pc.t.replaceReqCanceler(req.cancelKey, nil) { 2662 if debugRoundTrip { 2663 req.logf("closech recv: %T %#v", pc.closed, pc.closed) 2664 } 2665 return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) 2666 } 2667 case <-respHeaderTimer: 2668 if debugRoundTrip { 2669 req.logf("timeout waiting for response headers.") 2670 } 2671 pc.close(errTimeout) 2672 return nil, errTimeout 2673 case re := <-resc: 2674 if (re.res == nil) == (re.err == nil) { 2675 panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) 2676 } 2677 if debugRoundTrip { 2678 req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) 2679 } 2680 if re.err != nil { 2681 return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) 2682 } 2683 return re.res, nil 2684 case <-cancelChan: 2685 canceled = pc.t.cancelRequest(req.cancelKey, errRequestCanceled) 2686 cancelChan = nil 2687 case <-ctxDoneChan: 2688 canceled = pc.t.cancelRequest(req.cancelKey, req.Context().Err()) 2689 cancelChan = nil 2690 ctxDoneChan = nil 2691 } 2692 } 2693 } 2694 2695 // tLogKey is a context WithValue Key for test debugging contexts containing 2696 // a t.Logf func. See export_test.go's Request.WithT method. 2697 type tLogKey struct{} 2698 2699 func (tr *transportRequest) logf(format string, args ...any) { 2700 if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...any)); ok { 2701 logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) 2702 } 2703 } 2704 2705 // markReused marks this connection as having been successfully used for a 2706 // request and response. 2707 func (pc *persistConn) markReused() { 2708 pc.mu.Lock() 2709 pc.reused = true 2710 pc.mu.Unlock() 2711 } 2712 2713 // close closes the underlying TCP connection and closes 2714 // the pc.closech channel. 2715 // 2716 // The provided err is only for testing and debugging; in normal 2717 // circumstances it should never be seen by users. 2718 func (pc *persistConn) close(err error) { 2719 pc.mu.Lock() 2720 defer pc.mu.Unlock() 2721 pc.closeLocked(err) 2722 } 2723 2724 func (pc *persistConn) closeLocked(err error) { 2725 if err == nil { 2726 panic("nil error") 2727 } 2728 pc.broken = true 2729 if pc.closed == nil { 2730 pc.closed = err 2731 pc.t.decConnsPerHost(pc.cacheKey) 2732 // Close HTTP/1 (pc.alt == nil) connection. 2733 // HTTP/2 closes its connection itself. 2734 if pc.alt == nil { 2735 if err != errCallerOwnsConn { 2736 pc.conn.Close() 2737 } 2738 close(pc.closech) 2739 } 2740 } 2741 pc.mutateHeaderFunc = nil 2742 } 2743 2744 var portMap = map[string]string{ 2745 "http": "80", 2746 "https": "443", 2747 "socks5": "1080", 2748 } 2749 2750 func idnaASCIIFromURL(url *url.URL) string { 2751 addr := url.Hostname() 2752 if v, err := idnaASCII(addr); err == nil { 2753 addr = v 2754 } 2755 return addr 2756 } 2757 2758 // canonicalAddr returns url.Host but always with a ":port" suffix. 2759 func canonicalAddr(url *url.URL) string { 2760 port := url.Port() 2761 if port == "" { 2762 port = portMap[url.Scheme] 2763 } 2764 return net.JoinHostPort(idnaASCIIFromURL(url), port) 2765 } 2766 2767 // bodyEOFSignal is used by the HTTP/1 transport when reading response 2768 // bodies to make sure we see the end of a response body before 2769 // proceeding and reading on the connection again. 2770 // 2771 // It wraps a ReadCloser but runs fn (if non-nil) at most 2772 // once, right before its final (error-producing) Read or Close call 2773 // returns. fn should return the new error to return from Read or Close. 2774 // 2775 // If earlyCloseFn is non-nil and Close is called before io.EOF is 2776 // seen, earlyCloseFn is called instead of fn, and its return value is 2777 // the return value from Close. 2778 type bodyEOFSignal struct { 2779 body io.ReadCloser 2780 mu sync.Mutex // guards following 4 fields 2781 closed bool // whether Close has been called 2782 rerr error // sticky Read error 2783 fn func(error) error // err will be nil on Read io.EOF 2784 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 2785 } 2786 2787 var errReadOnClosedResBody = errors.New("http: read on closed response body") 2788 2789 func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 2790 es.mu.Lock() 2791 closed, rerr := es.closed, es.rerr 2792 es.mu.Unlock() 2793 if closed { 2794 return 0, errReadOnClosedResBody 2795 } 2796 if rerr != nil { 2797 return 0, rerr 2798 } 2799 2800 n, err = es.body.Read(p) 2801 if err != nil { 2802 es.mu.Lock() 2803 defer es.mu.Unlock() 2804 if es.rerr == nil { 2805 es.rerr = err 2806 } 2807 err = es.condfn(err) 2808 } 2809 return 2810 } 2811 2812 func (es *bodyEOFSignal) Close() error { 2813 es.mu.Lock() 2814 defer es.mu.Unlock() 2815 if es.closed { 2816 return nil 2817 } 2818 es.closed = true 2819 if es.earlyCloseFn != nil && es.rerr != io.EOF { 2820 return es.earlyCloseFn() 2821 } 2822 err := es.body.Close() 2823 return es.condfn(err) 2824 } 2825 2826 // caller must hold es.mu. 2827 func (es *bodyEOFSignal) condfn(err error) error { 2828 if es.fn == nil { 2829 return err 2830 } 2831 err = es.fn(err) 2832 es.fn = nil 2833 return err 2834 } 2835 2836 // gzipReader wraps a response body so it can lazily 2837 // call gzip.NewReader on the first call to Read 2838 type gzipReader struct { 2839 _ incomparable 2840 body *bodyEOFSignal // underlying HTTP/1 response body framing 2841 zr *gzip.Reader // lazily-initialized gzip reader 2842 zerr error // any error from gzip.NewReader; sticky 2843 } 2844 2845 func (gz *gzipReader) Read(p []byte) (n int, err error) { 2846 if gz.zr == nil { 2847 if gz.zerr == nil { 2848 gz.zr, gz.zerr = gzip.NewReader(gz.body) 2849 } 2850 if gz.zerr != nil { 2851 return 0, gz.zerr 2852 } 2853 } 2854 2855 gz.body.mu.Lock() 2856 if gz.body.closed { 2857 err = errReadOnClosedResBody 2858 } 2859 gz.body.mu.Unlock() 2860 2861 if err != nil { 2862 return 0, err 2863 } 2864 return gz.zr.Read(p) 2865 } 2866 2867 func (gz *gzipReader) Close() error { 2868 return gz.body.Close() 2869 } 2870 2871 type tlsHandshakeTimeoutError struct{} 2872 2873 func (tlsHandshakeTimeoutError) Timeout() bool { return true } 2874 func (tlsHandshakeTimeoutError) Temporary() bool { return true } 2875 func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 2876 2877 // fakeLocker is a sync.Locker which does nothing. It's used to guard 2878 // test-only fields when not under test, to avoid runtime atomic 2879 // overhead. 2880 type fakeLocker struct{} 2881 2882 func (fakeLocker) Lock() {} 2883 func (fakeLocker) Unlock() {} 2884 2885 // cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if 2886 // cfg is nil. This is safe to call even if cfg is in active use by a TLS 2887 // client or server. 2888 func cloneTLSConfig(cfg *tls.Config) *tls.Config { 2889 if cfg == nil { 2890 return &tls.Config{} 2891 } 2892 return cfg.Clone() 2893 } 2894 2895 type connLRU struct { 2896 ll *list.List // list.Element.Value type is of *persistConn 2897 m map[*persistConn]*list.Element 2898 } 2899 2900 // add adds pc to the head of the linked list. 2901 func (cl *connLRU) add(pc *persistConn) { 2902 if cl.ll == nil { 2903 cl.ll = list.New() 2904 cl.m = make(map[*persistConn]*list.Element) 2905 } 2906 ele := cl.ll.PushFront(pc) 2907 if _, ok := cl.m[pc]; ok { 2908 panic("persistConn was already in LRU") 2909 } 2910 cl.m[pc] = ele 2911 } 2912 2913 func (cl *connLRU) removeOldest() *persistConn { 2914 ele := cl.ll.Back() 2915 pc := ele.Value.(*persistConn) 2916 cl.ll.Remove(ele) 2917 delete(cl.m, pc) 2918 return pc 2919 } 2920 2921 // remove removes pc from cl. 2922 func (cl *connLRU) remove(pc *persistConn) { 2923 if ele, ok := cl.m[pc]; ok { 2924 cl.ll.Remove(ele) 2925 delete(cl.m, pc) 2926 } 2927 } 2928 2929 // len returns the number of items in the cache. 2930 func (cl *connLRU) len() int { 2931 return len(cl.m) 2932 }