github.com/megatontech/mynoteforgo@v0.0.0-20200507084910-5d0c6ea6e890/源码/net/http/transport.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client implementation. See RFC 7230 through 7235. 6 // 7 // This is the low-level Transport implementation of RoundTripper. 8 // The high-level interface is in client.go. 9 10 package http 11 12 import ( 13 "bufio" 14 "compress/gzip" 15 "container/list" 16 "context" 17 "crypto/tls" 18 "errors" 19 "fmt" 20 "io" 21 "log" 22 "net" 23 "net/http/httptrace" 24 "net/textproto" 25 "net/url" 26 "os" 27 "reflect" 28 "strings" 29 "sync" 30 "sync/atomic" 31 "time" 32 33 "internal/x/net/http/httpguts" 34 "internal/x/net/http/httpproxy" 35 ) 36 37 // DefaultTransport is the default implementation of Transport and is 38 // used by DefaultClient. It establishes network connections as needed 39 // and caches them for reuse by subsequent calls. It uses HTTP proxies 40 // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and 41 // $no_proxy) environment variables. 42 var DefaultTransport RoundTripper = &Transport{ 43 Proxy: ProxyFromEnvironment, 44 DialContext: (&net.Dialer{ 45 Timeout: 30 * time.Second, 46 KeepAlive: 30 * time.Second, 47 DualStack: true, 48 }).DialContext, 49 MaxIdleConns: 100, 50 IdleConnTimeout: 90 * time.Second, 51 TLSHandshakeTimeout: 10 * time.Second, 52 ExpectContinueTimeout: 1 * time.Second, 53 } 54 55 // DefaultMaxIdleConnsPerHost is the default value of Transport's 56 // MaxIdleConnsPerHost. 57 const DefaultMaxIdleConnsPerHost = 2 58 59 // connsPerHostClosedCh is a closed channel used by MaxConnsPerHost 60 // for the property that receives from a closed channel return the 61 // zero value. 62 var connsPerHostClosedCh = make(chan struct{}) 63 64 func init() { 65 close(connsPerHostClosedCh) 66 } 67 68 // Transport is an implementation of RoundTripper that supports HTTP, 69 // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 70 // 71 // By default, Transport caches connections for future re-use. 72 // This may leave many open connections when accessing many hosts. 73 // This behavior can be managed using Transport's CloseIdleConnections method 74 // and the MaxIdleConnsPerHost and DisableKeepAlives fields. 75 // 76 // Transports should be reused instead of created as needed. 77 // Transports are safe for concurrent use by multiple goroutines. 78 // 79 // A Transport is a low-level primitive for making HTTP and HTTPS requests. 80 // For high-level functionality, such as cookies and redirects, see Client. 81 // 82 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 83 // for HTTPS URLs, depending on whether the server supports HTTP/2, 84 // and how the Transport is configured. The DefaultTransport supports HTTP/2. 85 // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 86 // and call ConfigureTransport. See the package docs for more about HTTP/2. 87 // 88 // Responses with status codes in the 1xx range are either handled 89 // automatically (100 expect-continue) or ignored. The one 90 // exception is HTTP status code 101 (Switching Protocols), which is 91 // considered a terminal status and returned by RoundTrip. To see the 92 // ignored 1xx responses, use the httptrace trace package's 93 // ClientTrace.Got1xxResponse. 94 // 95 // Transport only retries a request upon encountering a network error 96 // if the request is idempotent and either has no body or has its 97 // Request.GetBody defined. HTTP requests are considered idempotent if 98 // they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their 99 // Header map contains an "Idempotency-Key" or "X-Idempotency-Key" 100 // entry. If the idempotency key value is an zero-length slice, the 101 // request is treated as idempotent but the header is not sent on the 102 // wire. 103 type Transport struct { 104 idleMu sync.Mutex 105 wantIdle bool // user has requested to close all idle conns 106 idleConn map[connectMethodKey][]*persistConn // most recently used at end 107 idleConnCh map[connectMethodKey]chan *persistConn 108 idleLRU connLRU 109 110 reqMu sync.Mutex 111 reqCanceler map[*Request]func(error) 112 113 altMu sync.Mutex // guards changing altProto only 114 altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme 115 116 connCountMu sync.Mutex 117 connPerHostCount map[connectMethodKey]int 118 connPerHostAvailable map[connectMethodKey]chan struct{} 119 120 // Proxy specifies a function to return a proxy for a given 121 // Request. If the function returns a non-nil error, the 122 // request is aborted with the provided error. 123 // 124 // The proxy type is determined by the URL scheme. "http", 125 // "https", and "socks5" are supported. If the scheme is empty, 126 // "http" is assumed. 127 // 128 // If Proxy is nil or returns a nil *URL, no proxy is used. 129 Proxy func(*Request) (*url.URL, error) 130 131 // DialContext specifies the dial function for creating unencrypted TCP connections. 132 // If DialContext is nil (and the deprecated Dial below is also nil), 133 // then the transport dials using package net. 134 // 135 // DialContext runs concurrently with calls to RoundTrip. 136 // A RoundTrip call that initiates a dial may end up using 137 // a connection dialed previously when the earlier connection 138 // becomes idle before the later DialContext completes. 139 DialContext func(ctx context.Context, network, addr string) (net.Conn, error) 140 141 // Dial specifies the dial function for creating unencrypted TCP connections. 142 // 143 // Dial runs concurrently with calls to RoundTrip. 144 // A RoundTrip call that initiates a dial may end up using 145 // a connection dialed previously when the earlier connection 146 // becomes idle before the later Dial completes. 147 // 148 // Deprecated: Use DialContext instead, which allows the transport 149 // to cancel dials as soon as they are no longer needed. 150 // If both are set, DialContext takes priority. 151 Dial func(network, addr string) (net.Conn, error) 152 153 // DialTLS specifies an optional dial function for creating 154 // TLS connections for non-proxied HTTPS requests. 155 // 156 // If DialTLS is nil, Dial and TLSClientConfig are used. 157 // 158 // If DialTLS is set, the Dial hook is not used for HTTPS 159 // requests and the TLSClientConfig and TLSHandshakeTimeout 160 // are ignored. The returned net.Conn is assumed to already be 161 // past the TLS handshake. 162 DialTLS func(network, addr string) (net.Conn, error) 163 164 // TLSClientConfig specifies the TLS configuration to use with 165 // tls.Client. 166 // If nil, the default configuration is used. 167 // If non-nil, HTTP/2 support may not be enabled by default. 168 TLSClientConfig *tls.Config 169 170 // TLSHandshakeTimeout specifies the maximum amount of time waiting to 171 // wait for a TLS handshake. Zero means no timeout. 172 TLSHandshakeTimeout time.Duration 173 174 // DisableKeepAlives, if true, disables HTTP keep-alives and 175 // will only use the connection to the server for a single 176 // HTTP request. 177 // 178 // This is unrelated to the similarly named TCP keep-alives. 179 DisableKeepAlives bool 180 181 // DisableCompression, if true, prevents the Transport from 182 // requesting compression with an "Accept-Encoding: gzip" 183 // request header when the Request contains no existing 184 // Accept-Encoding value. If the Transport requests gzip on 185 // its own and gets a gzipped response, it's transparently 186 // decoded in the Response.Body. However, if the user 187 // explicitly requested gzip it is not automatically 188 // uncompressed. 189 DisableCompression bool 190 191 // MaxIdleConns controls the maximum number of idle (keep-alive) 192 // connections across all hosts. Zero means no limit. 193 MaxIdleConns int 194 195 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 196 // (keep-alive) connections to keep per-host. If zero, 197 // DefaultMaxIdleConnsPerHost is used. 198 MaxIdleConnsPerHost int 199 200 // MaxConnsPerHost optionally limits the total number of 201 // connections per host, including connections in the dialing, 202 // active, and idle states. On limit violation, dials will block. 203 // 204 // Zero means no limit. 205 // 206 // For HTTP/2, this currently only controls the number of new 207 // connections being created at a time, instead of the total 208 // number. In practice, hosts using HTTP/2 only have about one 209 // idle connection, though. 210 MaxConnsPerHost int 211 212 // IdleConnTimeout is the maximum amount of time an idle 213 // (keep-alive) connection will remain idle before closing 214 // itself. 215 // Zero means no limit. 216 IdleConnTimeout time.Duration 217 218 // ResponseHeaderTimeout, if non-zero, specifies the amount of 219 // time to wait for a server's response headers after fully 220 // writing the request (including its body, if any). This 221 // time does not include the time to read the response body. 222 ResponseHeaderTimeout time.Duration 223 224 // ExpectContinueTimeout, if non-zero, specifies the amount of 225 // time to wait for a server's first response headers after fully 226 // writing the request headers if the request has an 227 // "Expect: 100-continue" header. Zero means no timeout and 228 // causes the body to be sent immediately, without 229 // waiting for the server to approve. 230 // This time does not include the time to send the request header. 231 ExpectContinueTimeout time.Duration 232 233 // TLSNextProto specifies how the Transport switches to an 234 // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN 235 // protocol negotiation. If Transport dials an TLS connection 236 // with a non-empty protocol name and TLSNextProto contains a 237 // map entry for that key (such as "h2"), then the func is 238 // called with the request's authority (such as "example.com" 239 // or "example.com:1234") and the TLS connection. The function 240 // must return a RoundTripper that then handles the request. 241 // If TLSNextProto is not nil, HTTP/2 support is not enabled 242 // automatically. 243 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 244 245 // ProxyConnectHeader optionally specifies headers to send to 246 // proxies during CONNECT requests. 247 ProxyConnectHeader Header 248 249 // MaxResponseHeaderBytes specifies a limit on how many 250 // response bytes are allowed in the server's response 251 // header. 252 // 253 // Zero means to use a default limit. 254 MaxResponseHeaderBytes int64 255 256 // nextProtoOnce guards initialization of TLSNextProto and 257 // h2transport (via onceSetNextProtoDefaults) 258 nextProtoOnce sync.Once 259 h2transport h2Transport // non-nil if http2 wired up 260 } 261 262 // h2Transport is the interface we expect to be able to call from 263 // net/http against an *http2.Transport that's either bundled into 264 // h2_bundle.go or supplied by the user via x/net/http2. 265 // 266 // We name it with the "h2" prefix to stay out of the "http2" prefix 267 // namespace used by x/tools/cmd/bundle for h2_bundle.go. 268 type h2Transport interface { 269 CloseIdleConnections() 270 } 271 272 // onceSetNextProtoDefaults initializes TLSNextProto. 273 // It must be called via t.nextProtoOnce.Do. 274 func (t *Transport) onceSetNextProtoDefaults() { 275 if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") { 276 return 277 } 278 279 // If they've already configured http2 with 280 // golang.org/x/net/http2 instead of the bundled copy, try to 281 // get at its http2.Transport value (via the "https" 282 // altproto map) so we can call CloseIdleConnections on it if 283 // requested. (Issue 22891) 284 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 285 if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 { 286 if v := rv.Field(0); v.CanInterface() { 287 if h2i, ok := v.Interface().(h2Transport); ok { 288 t.h2transport = h2i 289 return 290 } 291 } 292 } 293 294 if t.TLSNextProto != nil { 295 // This is the documented way to disable http2 on a 296 // Transport. 297 return 298 } 299 if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil { 300 // Be conservative and don't automatically enable 301 // http2 if they've specified a custom TLS config or 302 // custom dialers. Let them opt-in themselves via 303 // http2.ConfigureTransport so we don't surprise them 304 // by modifying their tls.Config. Issue 14275. 305 return 306 } 307 t2, err := http2configureTransport(t) 308 if err != nil { 309 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 310 return 311 } 312 t.h2transport = t2 313 314 // Auto-configure the http2.Transport's MaxHeaderListSize from 315 // the http.Transport's MaxResponseHeaderBytes. They don't 316 // exactly mean the same thing, but they're close. 317 // 318 // TODO: also add this to x/net/http2.Configure Transport, behind 319 // a +build go1.7 build tag: 320 if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { 321 const h2max = 1<<32 - 1 322 if limit1 >= h2max { 323 t2.MaxHeaderListSize = h2max 324 } else { 325 t2.MaxHeaderListSize = uint32(limit1) 326 } 327 } 328 } 329 330 // ProxyFromEnvironment returns the URL of the proxy to use for a 331 // given request, as indicated by the environment variables 332 // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 333 // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https 334 // requests. 335 // 336 // The environment values may be either a complete URL or a 337 // "host[:port]", in which case the "http" scheme is assumed. 338 // An error is returned if the value is a different form. 339 // 340 // A nil URL and nil error are returned if no proxy is defined in the 341 // environment, or a proxy should not be used for the given request, 342 // as defined by NO_PROXY. 343 // 344 // As a special case, if req.URL.Host is "localhost" (with or without 345 // a port number), then a nil URL and nil error will be returned. 346 func ProxyFromEnvironment(req *Request) (*url.URL, error) { 347 return envProxyFunc()(req.URL) 348 } 349 350 // ProxyURL returns a proxy function (for use in a Transport) 351 // that always returns the same URL. 352 func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 353 return func(*Request) (*url.URL, error) { 354 return fixedURL, nil 355 } 356 } 357 358 // transportRequest is a wrapper around a *Request that adds 359 // optional extra headers to write and stores any error to return 360 // from roundTrip. 361 type transportRequest struct { 362 *Request // original request, not to be mutated 363 extra Header // extra headers to write, or nil 364 trace *httptrace.ClientTrace // optional 365 366 mu sync.Mutex // guards err 367 err error // first setError value for mapRoundTripError to consider 368 } 369 370 func (tr *transportRequest) extraHeaders() Header { 371 if tr.extra == nil { 372 tr.extra = make(Header) 373 } 374 return tr.extra 375 } 376 377 func (tr *transportRequest) setError(err error) { 378 tr.mu.Lock() 379 if tr.err == nil { 380 tr.err = err 381 } 382 tr.mu.Unlock() 383 } 384 385 // useRegisteredProtocol reports whether an alternate protocol (as reqistered 386 // with Transport.RegisterProtocol) should be respected for this request. 387 func (t *Transport) useRegisteredProtocol(req *Request) bool { 388 if req.URL.Scheme == "https" && req.requiresHTTP1() { 389 // If this request requires HTTP/1, don't use the 390 // "https" alternate protocol, which is used by the 391 // HTTP/2 code to take over requests if there's an 392 // existing cached HTTP/2 connection. 393 return false 394 } 395 return true 396 } 397 398 // roundTrip implements a RoundTripper over HTTP. 399 func (t *Transport) roundTrip(req *Request) (*Response, error) { 400 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 401 ctx := req.Context() 402 trace := httptrace.ContextClientTrace(ctx) 403 404 if req.URL == nil { 405 req.closeBody() 406 return nil, errors.New("http: nil Request.URL") 407 } 408 if req.Header == nil { 409 req.closeBody() 410 return nil, errors.New("http: nil Request.Header") 411 } 412 scheme := req.URL.Scheme 413 isHTTP := scheme == "http" || scheme == "https" 414 if isHTTP { 415 for k, vv := range req.Header { 416 if !httpguts.ValidHeaderFieldName(k) { 417 return nil, fmt.Errorf("net/http: invalid header field name %q", k) 418 } 419 for _, v := range vv { 420 if !httpguts.ValidHeaderFieldValue(v) { 421 return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) 422 } 423 } 424 } 425 } 426 427 if t.useRegisteredProtocol(req) { 428 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 429 if altRT := altProto[scheme]; altRT != nil { 430 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 431 return resp, err 432 } 433 } 434 } 435 if !isHTTP { 436 req.closeBody() 437 return nil, &badStringError{"unsupported protocol scheme", scheme} 438 } 439 if req.Method != "" && !validMethod(req.Method) { 440 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 441 } 442 if req.URL.Host == "" { 443 req.closeBody() 444 return nil, errors.New("http: no Host in request URL") 445 } 446 447 for { 448 select { 449 case <-ctx.Done(): 450 req.closeBody() 451 return nil, ctx.Err() 452 default: 453 } 454 455 // treq gets modified by roundTrip, so we need to recreate for each retry. 456 treq := &transportRequest{Request: req, trace: trace} 457 cm, err := t.connectMethodForRequest(treq) 458 if err != nil { 459 req.closeBody() 460 return nil, err 461 } 462 463 // Get the cached or newly-created connection to either the 464 // host (for http or https), the http proxy, or the http proxy 465 // pre-CONNECTed to https server. In any case, we'll be ready 466 // to send it requests. 467 pconn, err := t.getConn(treq, cm) 468 if err != nil { 469 t.setReqCanceler(req, nil) 470 req.closeBody() 471 return nil, err 472 } 473 474 var resp *Response 475 if pconn.alt != nil { 476 // HTTP/2 path. 477 t.decHostConnCount(cm.key()) // don't count cached http2 conns toward conns per host 478 t.setReqCanceler(req, nil) // not cancelable with CancelRequest 479 resp, err = pconn.alt.RoundTrip(req) 480 } else { 481 resp, err = pconn.roundTrip(treq) 482 } 483 if err == nil { 484 return resp, nil 485 } 486 if !pconn.shouldRetryRequest(req, err) { 487 // Issue 16465: return underlying net.Conn.Read error from peek, 488 // as we've historically done. 489 if e, ok := err.(transportReadFromServerError); ok { 490 err = e.err 491 } 492 return nil, err 493 } 494 testHookRoundTripRetried() 495 496 // Rewind the body if we're able to. 497 if req.GetBody != nil { 498 newReq := *req 499 var err error 500 newReq.Body, err = req.GetBody() 501 if err != nil { 502 return nil, err 503 } 504 req = &newReq 505 } 506 } 507 } 508 509 // shouldRetryRequest reports whether we should retry sending a failed 510 // HTTP request on a new connection. The non-nil input error is the 511 // error from roundTrip. 512 func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { 513 if http2isNoCachedConnError(err) { 514 // Issue 16582: if the user started a bunch of 515 // requests at once, they can all pick the same conn 516 // and violate the server's max concurrent streams. 517 // Instead, match the HTTP/1 behavior for now and dial 518 // again to get a new TCP connection, rather than failing 519 // this request. 520 return true 521 } 522 if err == errMissingHost { 523 // User error. 524 return false 525 } 526 if !pc.isReused() { 527 // This was a fresh connection. There's no reason the server 528 // should've hung up on us. 529 // 530 // Also, if we retried now, we could loop forever 531 // creating new connections and retrying if the server 532 // is just hanging up on us because it doesn't like 533 // our request (as opposed to sending an error). 534 return false 535 } 536 if _, ok := err.(nothingWrittenError); ok { 537 // We never wrote anything, so it's safe to retry, if there's no body or we 538 // can "rewind" the body with GetBody. 539 return req.outgoingLength() == 0 || req.GetBody != nil 540 } 541 if !req.isReplayable() { 542 // Don't retry non-idempotent requests. 543 return false 544 } 545 if _, ok := err.(transportReadFromServerError); ok { 546 // We got some non-EOF net.Conn.Read failure reading 547 // the 1st response byte from the server. 548 return true 549 } 550 if err == errServerClosedIdle { 551 // The server replied with io.EOF while we were trying to 552 // read the response. Probably an unfortunately keep-alive 553 // timeout, just as the client was writing a request. 554 return true 555 } 556 return false // conservatively 557 } 558 559 // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 560 var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 561 562 // RegisterProtocol registers a new protocol with scheme. 563 // The Transport will pass requests using the given scheme to rt. 564 // It is rt's responsibility to simulate HTTP request semantics. 565 // 566 // RegisterProtocol can be used by other packages to provide 567 // implementations of protocol schemes like "ftp" or "file". 568 // 569 // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will 570 // handle the RoundTrip itself for that one request, as if the 571 // protocol were not registered. 572 func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 573 t.altMu.Lock() 574 defer t.altMu.Unlock() 575 oldMap, _ := t.altProto.Load().(map[string]RoundTripper) 576 if _, exists := oldMap[scheme]; exists { 577 panic("protocol " + scheme + " already registered") 578 } 579 newMap := make(map[string]RoundTripper) 580 for k, v := range oldMap { 581 newMap[k] = v 582 } 583 newMap[scheme] = rt 584 t.altProto.Store(newMap) 585 } 586 587 // CloseIdleConnections closes any connections which were previously 588 // connected from previous requests but are now sitting idle in 589 // a "keep-alive" state. It does not interrupt any connections currently 590 // in use. 591 func (t *Transport) CloseIdleConnections() { 592 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 593 t.idleMu.Lock() 594 m := t.idleConn 595 t.idleConn = nil 596 t.idleConnCh = nil 597 t.wantIdle = true 598 t.idleLRU = connLRU{} 599 t.idleMu.Unlock() 600 for _, conns := range m { 601 for _, pconn := range conns { 602 pconn.close(errCloseIdleConns) 603 } 604 } 605 if t2 := t.h2transport; t2 != nil { 606 t2.CloseIdleConnections() 607 } 608 } 609 610 // CancelRequest cancels an in-flight request by closing its connection. 611 // CancelRequest should only be called after RoundTrip has returned. 612 // 613 // Deprecated: Use Request.WithContext to create a request with a 614 // cancelable context instead. CancelRequest cannot cancel HTTP/2 615 // requests. 616 func (t *Transport) CancelRequest(req *Request) { 617 t.cancelRequest(req, errRequestCanceled) 618 } 619 620 // Cancel an in-flight request, recording the error value. 621 func (t *Transport) cancelRequest(req *Request, err error) { 622 t.reqMu.Lock() 623 cancel := t.reqCanceler[req] 624 delete(t.reqCanceler, req) 625 t.reqMu.Unlock() 626 if cancel != nil { 627 cancel(err) 628 } 629 } 630 631 // 632 // Private implementation past this point. 633 // 634 635 var ( 636 // proxyConfigOnce guards proxyConfig 637 envProxyOnce sync.Once 638 envProxyFuncValue func(*url.URL) (*url.URL, error) 639 ) 640 641 // defaultProxyConfig returns a ProxyConfig value looked up 642 // from the environment. This mitigates expensive lookups 643 // on some platforms (e.g. Windows). 644 func envProxyFunc() func(*url.URL) (*url.URL, error) { 645 envProxyOnce.Do(func() { 646 envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() 647 }) 648 return envProxyFuncValue 649 } 650 651 // resetProxyConfig is used by tests. 652 func resetProxyConfig() { 653 envProxyOnce = sync.Once{} 654 envProxyFuncValue = nil 655 } 656 657 func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 658 if port := treq.URL.Port(); !validPort(port) { 659 return cm, fmt.Errorf("invalid URL port %q", port) 660 } 661 cm.targetScheme = treq.URL.Scheme 662 cm.targetAddr = canonicalAddr(treq.URL) 663 if t.Proxy != nil { 664 cm.proxyURL, err = t.Proxy(treq.Request) 665 if err == nil && cm.proxyURL != nil { 666 if port := cm.proxyURL.Port(); !validPort(port) { 667 return cm, fmt.Errorf("invalid proxy URL port %q", port) 668 } 669 } 670 } 671 cm.onlyH1 = treq.requiresHTTP1() 672 return cm, err 673 } 674 675 // proxyAuth returns the Proxy-Authorization header to set 676 // on requests, if applicable. 677 func (cm *connectMethod) proxyAuth() string { 678 if cm.proxyURL == nil { 679 return "" 680 } 681 if u := cm.proxyURL.User; u != nil { 682 username := u.Username() 683 password, _ := u.Password() 684 return "Basic " + basicAuth(username, password) 685 } 686 return "" 687 } 688 689 // error values for debugging and testing, not seen by users. 690 var ( 691 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") 692 errConnBroken = errors.New("http: putIdleConn: connection is in bad state") 693 errWantIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") 694 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") 695 errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") 696 errCloseIdleConns = errors.New("http: CloseIdleConnections called") 697 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") 698 errIdleConnTimeout = errors.New("http: idle connection timeout") 699 errNotCachingH2Conn = errors.New("http: not caching alternate protocol's connections") 700 701 // errServerClosedIdle is not seen by users for idempotent requests, but may be 702 // seen by a user if the server shuts down an idle connection and sends its FIN 703 // in flight with already-written POST body bytes from the client. 704 // See https://github.com/golang/go/issues/19943#issuecomment-355607646 705 errServerClosedIdle = errors.New("http: server closed idle connection") 706 ) 707 708 // transportReadFromServerError is used by Transport.readLoop when the 709 // 1 byte peek read fails and we're actually anticipating a response. 710 // Usually this is just due to the inherent keep-alive shut down race, 711 // where the server closed the connection at the same time the client 712 // wrote. The underlying err field is usually io.EOF or some 713 // ECONNRESET sort of thing which varies by platform. But it might be 714 // the user's custom net.Conn.Read error too, so we carry it along for 715 // them to return from Transport.RoundTrip. 716 type transportReadFromServerError struct { 717 err error 718 } 719 720 func (e transportReadFromServerError) Error() string { 721 return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) 722 } 723 724 func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { 725 if err := t.tryPutIdleConn(pconn); err != nil { 726 pconn.close(err) 727 } 728 } 729 730 func (t *Transport) maxIdleConnsPerHost() int { 731 if v := t.MaxIdleConnsPerHost; v != 0 { 732 return v 733 } 734 return DefaultMaxIdleConnsPerHost 735 } 736 737 // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting 738 // a new request. 739 // If pconn is no longer needed or not in a good state, tryPutIdleConn returns 740 // an error explaining why it wasn't registered. 741 // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. 742 func (t *Transport) tryPutIdleConn(pconn *persistConn) error { 743 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 744 return errKeepAlivesDisabled 745 } 746 if pconn.isBroken() { 747 return errConnBroken 748 } 749 if pconn.alt != nil { 750 return errNotCachingH2Conn 751 } 752 pconn.markReused() 753 key := pconn.cacheKey 754 755 t.idleMu.Lock() 756 defer t.idleMu.Unlock() 757 758 waitingDialer := t.idleConnCh[key] 759 select { 760 case waitingDialer <- pconn: 761 // We're done with this pconn and somebody else is 762 // currently waiting for a conn of this type (they're 763 // actively dialing, but this conn is ready 764 // first). Chrome calls this socket late binding. See 765 // https://insouciant.org/tech/connection-management-in-chromium/ 766 return nil 767 default: 768 if waitingDialer != nil { 769 // They had populated this, but their dial won 770 // first, so we can clean up this map entry. 771 delete(t.idleConnCh, key) 772 } 773 } 774 if t.wantIdle { 775 return errWantIdle 776 } 777 if t.idleConn == nil { 778 t.idleConn = make(map[connectMethodKey][]*persistConn) 779 } 780 idles := t.idleConn[key] 781 if len(idles) >= t.maxIdleConnsPerHost() { 782 return errTooManyIdleHost 783 } 784 for _, exist := range idles { 785 if exist == pconn { 786 log.Fatalf("dup idle pconn %p in freelist", pconn) 787 } 788 } 789 t.idleConn[key] = append(idles, pconn) 790 t.idleLRU.add(pconn) 791 if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { 792 oldest := t.idleLRU.removeOldest() 793 oldest.close(errTooManyIdle) 794 t.removeIdleConnLocked(oldest) 795 } 796 if t.IdleConnTimeout > 0 { 797 if pconn.idleTimer != nil { 798 pconn.idleTimer.Reset(t.IdleConnTimeout) 799 } else { 800 pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) 801 } 802 } 803 pconn.idleAt = time.Now() 804 return nil 805 } 806 807 // getIdleConnCh returns a channel to receive and return idle 808 // persistent connection for the given connectMethod. 809 // It may return nil, if persistent connections are not being used. 810 func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { 811 if t.DisableKeepAlives { 812 return nil 813 } 814 key := cm.key() 815 t.idleMu.Lock() 816 defer t.idleMu.Unlock() 817 t.wantIdle = false 818 if t.idleConnCh == nil { 819 t.idleConnCh = make(map[connectMethodKey]chan *persistConn) 820 } 821 ch, ok := t.idleConnCh[key] 822 if !ok { 823 ch = make(chan *persistConn) 824 t.idleConnCh[key] = ch 825 } 826 return ch 827 } 828 829 func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn, idleSince time.Time) { 830 key := cm.key() 831 t.idleMu.Lock() 832 defer t.idleMu.Unlock() 833 for { 834 pconns, ok := t.idleConn[key] 835 if !ok { 836 return nil, time.Time{} 837 } 838 if len(pconns) == 1 { 839 pconn = pconns[0] 840 delete(t.idleConn, key) 841 } else { 842 // 2 or more cached connections; use the most 843 // recently used one at the end. 844 pconn = pconns[len(pconns)-1] 845 t.idleConn[key] = pconns[:len(pconns)-1] 846 } 847 t.idleLRU.remove(pconn) 848 if pconn.isBroken() { 849 // There is a tiny window where this is 850 // possible, between the connecting dying and 851 // the persistConn readLoop calling 852 // Transport.removeIdleConn. Just skip it and 853 // carry on. 854 continue 855 } 856 return pconn, pconn.idleAt 857 } 858 } 859 860 // removeIdleConn marks pconn as dead. 861 func (t *Transport) removeIdleConn(pconn *persistConn) { 862 t.idleMu.Lock() 863 defer t.idleMu.Unlock() 864 t.removeIdleConnLocked(pconn) 865 } 866 867 // t.idleMu must be held. 868 func (t *Transport) removeIdleConnLocked(pconn *persistConn) { 869 if pconn.idleTimer != nil { 870 pconn.idleTimer.Stop() 871 } 872 t.idleLRU.remove(pconn) 873 key := pconn.cacheKey 874 pconns := t.idleConn[key] 875 switch len(pconns) { 876 case 0: 877 // Nothing 878 case 1: 879 if pconns[0] == pconn { 880 delete(t.idleConn, key) 881 } 882 default: 883 for i, v := range pconns { 884 if v != pconn { 885 continue 886 } 887 // Slide down, keeping most recently-used 888 // conns at the end. 889 copy(pconns[i:], pconns[i+1:]) 890 t.idleConn[key] = pconns[:len(pconns)-1] 891 break 892 } 893 } 894 } 895 896 func (t *Transport) setReqCanceler(r *Request, fn func(error)) { 897 t.reqMu.Lock() 898 defer t.reqMu.Unlock() 899 if t.reqCanceler == nil { 900 t.reqCanceler = make(map[*Request]func(error)) 901 } 902 if fn != nil { 903 t.reqCanceler[r] = fn 904 } else { 905 delete(t.reqCanceler, r) 906 } 907 } 908 909 // replaceReqCanceler replaces an existing cancel function. If there is no cancel function 910 // for the request, we don't set the function and return false. 911 // Since CancelRequest will clear the canceler, we can use the return value to detect if 912 // the request was canceled since the last setReqCancel call. 913 func (t *Transport) replaceReqCanceler(r *Request, fn func(error)) bool { 914 t.reqMu.Lock() 915 defer t.reqMu.Unlock() 916 _, ok := t.reqCanceler[r] 917 if !ok { 918 return false 919 } 920 if fn != nil { 921 t.reqCanceler[r] = fn 922 } else { 923 delete(t.reqCanceler, r) 924 } 925 return true 926 } 927 928 var zeroDialer net.Dialer 929 930 func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { 931 if t.DialContext != nil { 932 return t.DialContext(ctx, network, addr) 933 } 934 if t.Dial != nil { 935 c, err := t.Dial(network, addr) 936 if c == nil && err == nil { 937 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") 938 } 939 return c, err 940 } 941 return zeroDialer.DialContext(ctx, network, addr) 942 } 943 944 // getConn dials and creates a new persistConn to the target as 945 // specified in the connectMethod. This includes doing a proxy CONNECT 946 // and/or setting up TLS. If this doesn't return an error, the persistConn 947 // is ready to write requests to. 948 func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) { 949 req := treq.Request 950 trace := treq.trace 951 ctx := req.Context() 952 if trace != nil && trace.GetConn != nil { 953 trace.GetConn(cm.addr()) 954 } 955 if pc, idleSince := t.getIdleConn(cm); pc != nil { 956 if trace != nil && trace.GotConn != nil { 957 trace.GotConn(pc.gotIdleConnTrace(idleSince)) 958 } 959 // set request canceler to some non-nil function so we 960 // can detect whether it was cleared between now and when 961 // we enter roundTrip 962 t.setReqCanceler(req, func(error) {}) 963 return pc, nil 964 } 965 966 type dialRes struct { 967 pc *persistConn 968 err error 969 } 970 dialc := make(chan dialRes) 971 cmKey := cm.key() 972 973 // Copy these hooks so we don't race on the postPendingDial in 974 // the goroutine we launch. Issue 11136. 975 testHookPrePendingDial := testHookPrePendingDial 976 testHookPostPendingDial := testHookPostPendingDial 977 978 handlePendingDial := func() { 979 testHookPrePendingDial() 980 go func() { 981 if v := <-dialc; v.err == nil { 982 t.putOrCloseIdleConn(v.pc) 983 } else { 984 t.decHostConnCount(cmKey) 985 } 986 testHookPostPendingDial() 987 }() 988 } 989 990 cancelc := make(chan error, 1) 991 t.setReqCanceler(req, func(err error) { cancelc <- err }) 992 993 if t.MaxConnsPerHost > 0 { 994 select { 995 case <-t.incHostConnCount(cmKey): 996 // count below conn per host limit; proceed 997 case pc := <-t.getIdleConnCh(cm): 998 if trace != nil && trace.GotConn != nil { 999 trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()}) 1000 } 1001 return pc, nil 1002 case <-req.Cancel: 1003 return nil, errRequestCanceledConn 1004 case <-req.Context().Done(): 1005 return nil, req.Context().Err() 1006 case err := <-cancelc: 1007 if err == errRequestCanceled { 1008 err = errRequestCanceledConn 1009 } 1010 return nil, err 1011 } 1012 } 1013 1014 go func() { 1015 pc, err := t.dialConn(ctx, cm) 1016 dialc <- dialRes{pc, err} 1017 }() 1018 1019 idleConnCh := t.getIdleConnCh(cm) 1020 select { 1021 case v := <-dialc: 1022 // Our dial finished. 1023 if v.pc != nil { 1024 if trace != nil && trace.GotConn != nil && v.pc.alt == nil { 1025 trace.GotConn(httptrace.GotConnInfo{Conn: v.pc.conn}) 1026 } 1027 return v.pc, nil 1028 } 1029 // Our dial failed. See why to return a nicer error 1030 // value. 1031 t.decHostConnCount(cmKey) 1032 select { 1033 case <-req.Cancel: 1034 // It was an error due to cancelation, so prioritize that 1035 // error value. (Issue 16049) 1036 return nil, errRequestCanceledConn 1037 case <-req.Context().Done(): 1038 return nil, req.Context().Err() 1039 case err := <-cancelc: 1040 if err == errRequestCanceled { 1041 err = errRequestCanceledConn 1042 } 1043 return nil, err 1044 default: 1045 // It wasn't an error due to cancelation, so 1046 // return the original error message: 1047 return nil, v.err 1048 } 1049 case pc := <-idleConnCh: 1050 // Another request finished first and its net.Conn 1051 // became available before our dial. Or somebody 1052 // else's dial that they didn't use. 1053 // But our dial is still going, so give it away 1054 // when it finishes: 1055 handlePendingDial() 1056 if trace != nil && trace.GotConn != nil { 1057 trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()}) 1058 } 1059 return pc, nil 1060 case <-req.Cancel: 1061 handlePendingDial() 1062 return nil, errRequestCanceledConn 1063 case <-req.Context().Done(): 1064 handlePendingDial() 1065 return nil, req.Context().Err() 1066 case err := <-cancelc: 1067 handlePendingDial() 1068 if err == errRequestCanceled { 1069 err = errRequestCanceledConn 1070 } 1071 return nil, err 1072 } 1073 } 1074 1075 // incHostConnCount increments the count of connections for a 1076 // given host. It returns an already-closed channel if the count 1077 // is not at its limit; otherwise it returns a channel which is 1078 // notified when the count is below the limit. 1079 func (t *Transport) incHostConnCount(cmKey connectMethodKey) <-chan struct{} { 1080 if t.MaxConnsPerHost <= 0 { 1081 return connsPerHostClosedCh 1082 } 1083 t.connCountMu.Lock() 1084 defer t.connCountMu.Unlock() 1085 if t.connPerHostCount[cmKey] == t.MaxConnsPerHost { 1086 if t.connPerHostAvailable == nil { 1087 t.connPerHostAvailable = make(map[connectMethodKey]chan struct{}) 1088 } 1089 ch, ok := t.connPerHostAvailable[cmKey] 1090 if !ok { 1091 ch = make(chan struct{}) 1092 t.connPerHostAvailable[cmKey] = ch 1093 } 1094 return ch 1095 } 1096 if t.connPerHostCount == nil { 1097 t.connPerHostCount = make(map[connectMethodKey]int) 1098 } 1099 t.connPerHostCount[cmKey]++ 1100 // return a closed channel to avoid race: if decHostConnCount is called 1101 // after incHostConnCount and during the nil check, decHostConnCount 1102 // will delete the channel since it's not being listened on yet. 1103 return connsPerHostClosedCh 1104 } 1105 1106 // decHostConnCount decrements the count of connections 1107 // for a given host. 1108 // See Transport.MaxConnsPerHost. 1109 func (t *Transport) decHostConnCount(cmKey connectMethodKey) { 1110 if t.MaxConnsPerHost <= 0 { 1111 return 1112 } 1113 t.connCountMu.Lock() 1114 defer t.connCountMu.Unlock() 1115 t.connPerHostCount[cmKey]-- 1116 select { 1117 case t.connPerHostAvailable[cmKey] <- struct{}{}: 1118 default: 1119 // close channel before deleting avoids getConn waiting forever in 1120 // case getConn has reference to channel but hasn't started waiting. 1121 // This could lead to more than MaxConnsPerHost in the unlikely case 1122 // that > 1 go routine has fetched the channel but none started waiting. 1123 if t.connPerHostAvailable[cmKey] != nil { 1124 close(t.connPerHostAvailable[cmKey]) 1125 } 1126 delete(t.connPerHostAvailable, cmKey) 1127 } 1128 if t.connPerHostCount[cmKey] == 0 { 1129 delete(t.connPerHostCount, cmKey) 1130 } 1131 } 1132 1133 // connCloseListener wraps a connection, the transport that dialed it 1134 // and the connected-to host key so the host connection count can be 1135 // transparently decremented by whatever closes the embedded connection. 1136 type connCloseListener struct { 1137 net.Conn 1138 t *Transport 1139 cmKey connectMethodKey 1140 didClose int32 1141 } 1142 1143 func (c *connCloseListener) Close() error { 1144 if atomic.AddInt32(&c.didClose, 1) != 1 { 1145 return nil 1146 } 1147 err := c.Conn.Close() 1148 c.t.decHostConnCount(c.cmKey) 1149 return err 1150 } 1151 1152 // The connect method and the transport can both specify a TLS 1153 // Host name. The transport's name takes precedence if present. 1154 func chooseTLSHost(cm connectMethod, t *Transport) string { 1155 tlsHost := "" 1156 if t.TLSClientConfig != nil { 1157 tlsHost = t.TLSClientConfig.ServerName 1158 } 1159 if tlsHost == "" { 1160 tlsHost = cm.tlsHost() 1161 } 1162 return tlsHost 1163 } 1164 1165 // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS 1166 // tunnel, this function establishes a nested TLS session inside the encrypted channel. 1167 // The remote endpoint's name may be overridden by TLSClientConfig.ServerName. 1168 func (pconn *persistConn) addTLS(name string, trace *httptrace.ClientTrace) error { 1169 // Initiate TLS and check remote host name against certificate. 1170 cfg := cloneTLSConfig(pconn.t.TLSClientConfig) 1171 if cfg.ServerName == "" { 1172 cfg.ServerName = name 1173 } 1174 if pconn.cacheKey.onlyH1 { 1175 cfg.NextProtos = nil 1176 } 1177 plainConn := pconn.conn 1178 tlsConn := tls.Client(plainConn, cfg) 1179 errc := make(chan error, 2) 1180 var timer *time.Timer // for canceling TLS handshake 1181 if d := pconn.t.TLSHandshakeTimeout; d != 0 { 1182 timer = time.AfterFunc(d, func() { 1183 errc <- tlsHandshakeTimeoutError{} 1184 }) 1185 } 1186 go func() { 1187 if trace != nil && trace.TLSHandshakeStart != nil { 1188 trace.TLSHandshakeStart() 1189 } 1190 err := tlsConn.Handshake() 1191 if timer != nil { 1192 timer.Stop() 1193 } 1194 errc <- err 1195 }() 1196 if err := <-errc; err != nil { 1197 plainConn.Close() 1198 if trace != nil && trace.TLSHandshakeDone != nil { 1199 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1200 } 1201 return err 1202 } 1203 cs := tlsConn.ConnectionState() 1204 if trace != nil && trace.TLSHandshakeDone != nil { 1205 trace.TLSHandshakeDone(cs, nil) 1206 } 1207 pconn.tlsState = &cs 1208 pconn.conn = tlsConn 1209 return nil 1210 } 1211 1212 func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) { 1213 pconn := &persistConn{ 1214 t: t, 1215 cacheKey: cm.key(), 1216 reqch: make(chan requestAndChan, 1), 1217 writech: make(chan writeRequest, 1), 1218 closech: make(chan struct{}), 1219 writeErrCh: make(chan error, 1), 1220 writeLoopDone: make(chan struct{}), 1221 } 1222 trace := httptrace.ContextClientTrace(ctx) 1223 wrapErr := func(err error) error { 1224 if cm.proxyURL != nil { 1225 // Return a typed error, per Issue 16997 1226 return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} 1227 } 1228 return err 1229 } 1230 if cm.scheme() == "https" && t.DialTLS != nil { 1231 var err error 1232 pconn.conn, err = t.DialTLS("tcp", cm.addr()) 1233 if err != nil { 1234 return nil, wrapErr(err) 1235 } 1236 if pconn.conn == nil { 1237 return nil, wrapErr(errors.New("net/http: Transport.DialTLS returned (nil, nil)")) 1238 } 1239 if tc, ok := pconn.conn.(*tls.Conn); ok { 1240 // Handshake here, in case DialTLS didn't. TLSNextProto below 1241 // depends on it for knowing the connection state. 1242 if trace != nil && trace.TLSHandshakeStart != nil { 1243 trace.TLSHandshakeStart() 1244 } 1245 if err := tc.Handshake(); err != nil { 1246 go pconn.conn.Close() 1247 if trace != nil && trace.TLSHandshakeDone != nil { 1248 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1249 } 1250 return nil, err 1251 } 1252 cs := tc.ConnectionState() 1253 if trace != nil && trace.TLSHandshakeDone != nil { 1254 trace.TLSHandshakeDone(cs, nil) 1255 } 1256 pconn.tlsState = &cs 1257 } 1258 } else { 1259 conn, err := t.dial(ctx, "tcp", cm.addr()) 1260 if err != nil { 1261 return nil, wrapErr(err) 1262 } 1263 pconn.conn = conn 1264 if cm.scheme() == "https" { 1265 var firstTLSHost string 1266 if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { 1267 return nil, wrapErr(err) 1268 } 1269 if err = pconn.addTLS(firstTLSHost, trace); err != nil { 1270 return nil, wrapErr(err) 1271 } 1272 } 1273 } 1274 1275 // Proxy setup. 1276 switch { 1277 case cm.proxyURL == nil: 1278 // Do nothing. Not using a proxy. 1279 case cm.proxyURL.Scheme == "socks5": 1280 conn := pconn.conn 1281 d := socksNewDialer("tcp", conn.RemoteAddr().String()) 1282 if u := cm.proxyURL.User; u != nil { 1283 auth := &socksUsernamePassword{ 1284 Username: u.Username(), 1285 } 1286 auth.Password, _ = u.Password() 1287 d.AuthMethods = []socksAuthMethod{ 1288 socksAuthMethodNotRequired, 1289 socksAuthMethodUsernamePassword, 1290 } 1291 d.Authenticate = auth.Authenticate 1292 } 1293 if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { 1294 conn.Close() 1295 return nil, err 1296 } 1297 case cm.targetScheme == "http": 1298 pconn.isProxy = true 1299 if pa := cm.proxyAuth(); pa != "" { 1300 pconn.mutateHeaderFunc = func(h Header) { 1301 h.Set("Proxy-Authorization", pa) 1302 } 1303 } 1304 case cm.targetScheme == "https": 1305 conn := pconn.conn 1306 hdr := t.ProxyConnectHeader 1307 if hdr == nil { 1308 hdr = make(Header) 1309 } 1310 connectReq := &Request{ 1311 Method: "CONNECT", 1312 URL: &url.URL{Opaque: cm.targetAddr}, 1313 Host: cm.targetAddr, 1314 Header: hdr, 1315 } 1316 if pa := cm.proxyAuth(); pa != "" { 1317 connectReq.Header.Set("Proxy-Authorization", pa) 1318 } 1319 connectReq.Write(conn) 1320 1321 // Read response. 1322 // Okay to use and discard buffered reader here, because 1323 // TLS server will not speak until spoken to. 1324 br := bufio.NewReader(conn) 1325 resp, err := ReadResponse(br, connectReq) 1326 if err != nil { 1327 conn.Close() 1328 return nil, err 1329 } 1330 if resp.StatusCode != 200 { 1331 f := strings.SplitN(resp.Status, " ", 2) 1332 conn.Close() 1333 if len(f) < 2 { 1334 return nil, errors.New("unknown status code") 1335 } 1336 return nil, errors.New(f[1]) 1337 } 1338 } 1339 1340 if cm.proxyURL != nil && cm.targetScheme == "https" { 1341 if err := pconn.addTLS(cm.tlsHost(), trace); err != nil { 1342 return nil, err 1343 } 1344 } 1345 1346 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 1347 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 1348 return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil 1349 } 1350 } 1351 1352 if t.MaxConnsPerHost > 0 { 1353 pconn.conn = &connCloseListener{Conn: pconn.conn, t: t, cmKey: pconn.cacheKey} 1354 } 1355 pconn.br = bufio.NewReader(pconn) 1356 pconn.bw = bufio.NewWriter(persistConnWriter{pconn}) 1357 go pconn.readLoop() 1358 go pconn.writeLoop() 1359 return pconn, nil 1360 } 1361 1362 // persistConnWriter is the io.Writer written to by pc.bw. 1363 // It accumulates the number of bytes written to the underlying conn, 1364 // so the retry logic can determine whether any bytes made it across 1365 // the wire. 1366 // This is exactly 1 pointer field wide so it can go into an interface 1367 // without allocation. 1368 type persistConnWriter struct { 1369 pc *persistConn 1370 } 1371 1372 func (w persistConnWriter) Write(p []byte) (n int, err error) { 1373 n, err = w.pc.conn.Write(p) 1374 w.pc.nwrite += int64(n) 1375 return 1376 } 1377 1378 // connectMethod is the map key (in its String form) for keeping persistent 1379 // TCP connections alive for subsequent HTTP requests. 1380 // 1381 // A connect method may be of the following types: 1382 // 1383 // connectMethod.key().String() Description 1384 // ------------------------------ ------------------------- 1385 // |http|foo.com http directly to server, no proxy 1386 // |https|foo.com https directly to server, no proxy 1387 // |https,h1|foo.com https directly to server w/o HTTP/2, no proxy 1388 // http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 1389 // http://proxy.com|http http to proxy, http to anywhere after that 1390 // socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com 1391 // socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com 1392 // https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com 1393 // https://proxy.com|http https to proxy, http to anywhere after that 1394 // 1395 type connectMethod struct { 1396 proxyURL *url.URL // nil for no proxy, else full proxy URL 1397 targetScheme string // "http" or "https" 1398 // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), 1399 // then targetAddr is not included in the connect method key, because the socket can 1400 // be reused for different targetAddr values. 1401 targetAddr string 1402 onlyH1 bool // whether to disable HTTP/2 and force HTTP/1 1403 } 1404 1405 func (cm *connectMethod) key() connectMethodKey { 1406 proxyStr := "" 1407 targetAddr := cm.targetAddr 1408 if cm.proxyURL != nil { 1409 proxyStr = cm.proxyURL.String() 1410 if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { 1411 targetAddr = "" 1412 } 1413 } 1414 return connectMethodKey{ 1415 proxy: proxyStr, 1416 scheme: cm.targetScheme, 1417 addr: targetAddr, 1418 onlyH1: cm.onlyH1, 1419 } 1420 } 1421 1422 // scheme returns the first hop scheme: http, https, or socks5 1423 func (cm *connectMethod) scheme() string { 1424 if cm.proxyURL != nil { 1425 return cm.proxyURL.Scheme 1426 } 1427 return cm.targetScheme 1428 } 1429 1430 // addr returns the first hop "host:port" to which we need to TCP connect. 1431 func (cm *connectMethod) addr() string { 1432 if cm.proxyURL != nil { 1433 return canonicalAddr(cm.proxyURL) 1434 } 1435 return cm.targetAddr 1436 } 1437 1438 // tlsHost returns the host name to match against the peer's 1439 // TLS certificate. 1440 func (cm *connectMethod) tlsHost() string { 1441 h := cm.targetAddr 1442 if hasPort(h) { 1443 h = h[:strings.LastIndex(h, ":")] 1444 } 1445 return h 1446 } 1447 1448 // connectMethodKey is the map key version of connectMethod, with a 1449 // stringified proxy URL (or the empty string) instead of a pointer to 1450 // a URL. 1451 type connectMethodKey struct { 1452 proxy, scheme, addr string 1453 onlyH1 bool 1454 } 1455 1456 func (k connectMethodKey) String() string { 1457 // Only used by tests. 1458 var h1 string 1459 if k.onlyH1 { 1460 h1 = ",h1" 1461 } 1462 return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr) 1463 } 1464 1465 // persistConn wraps a connection, usually a persistent one 1466 // (but may be used for non-keep-alive requests as well) 1467 type persistConn struct { 1468 // alt optionally specifies the TLS NextProto RoundTripper. 1469 // This is used for HTTP/2 today and future protocols later. 1470 // If it's non-nil, the rest of the fields are unused. 1471 alt RoundTripper 1472 1473 t *Transport 1474 cacheKey connectMethodKey 1475 conn net.Conn 1476 tlsState *tls.ConnectionState 1477 br *bufio.Reader // from conn 1478 bw *bufio.Writer // to conn 1479 nwrite int64 // bytes written 1480 reqch chan requestAndChan // written by roundTrip; read by readLoop 1481 writech chan writeRequest // written by roundTrip; read by writeLoop 1482 closech chan struct{} // closed when conn closed 1483 isProxy bool 1484 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 1485 readLimit int64 // bytes allowed to be read; owned by readLoop 1486 // writeErrCh passes the request write error (usually nil) 1487 // from the writeLoop goroutine to the readLoop which passes 1488 // it off to the res.Body reader, which then uses it to decide 1489 // whether or not a connection can be reused. Issue 7569. 1490 writeErrCh chan error 1491 1492 writeLoopDone chan struct{} // closed when write loop ends 1493 1494 // Both guarded by Transport.idleMu: 1495 idleAt time.Time // time it last become idle 1496 idleTimer *time.Timer // holding an AfterFunc to close it 1497 1498 mu sync.Mutex // guards following fields 1499 numExpectedResponses int 1500 closed error // set non-nil when conn is closed, before closech is closed 1501 canceledErr error // set non-nil if conn is canceled 1502 broken bool // an error has happened on this connection; marked broken so it's not reused. 1503 reused bool // whether conn has had successful request/response and is being reused. 1504 // mutateHeaderFunc is an optional func to modify extra 1505 // headers on each outbound request before it's written. (the 1506 // original Request given to RoundTrip is not modified) 1507 mutateHeaderFunc func(Header) 1508 } 1509 1510 func (pc *persistConn) maxHeaderResponseSize() int64 { 1511 if v := pc.t.MaxResponseHeaderBytes; v != 0 { 1512 return v 1513 } 1514 return 10 << 20 // conservative default; same as http2 1515 } 1516 1517 func (pc *persistConn) Read(p []byte) (n int, err error) { 1518 if pc.readLimit <= 0 { 1519 return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) 1520 } 1521 if int64(len(p)) > pc.readLimit { 1522 p = p[:pc.readLimit] 1523 } 1524 n, err = pc.conn.Read(p) 1525 if err == io.EOF { 1526 pc.sawEOF = true 1527 } 1528 pc.readLimit -= int64(n) 1529 return 1530 } 1531 1532 // isBroken reports whether this connection is in a known broken state. 1533 func (pc *persistConn) isBroken() bool { 1534 pc.mu.Lock() 1535 b := pc.closed != nil 1536 pc.mu.Unlock() 1537 return b 1538 } 1539 1540 // canceled returns non-nil if the connection was closed due to 1541 // CancelRequest or due to context cancelation. 1542 func (pc *persistConn) canceled() error { 1543 pc.mu.Lock() 1544 defer pc.mu.Unlock() 1545 return pc.canceledErr 1546 } 1547 1548 // isReused reports whether this connection is in a known broken state. 1549 func (pc *persistConn) isReused() bool { 1550 pc.mu.Lock() 1551 r := pc.reused 1552 pc.mu.Unlock() 1553 return r 1554 } 1555 1556 func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { 1557 pc.mu.Lock() 1558 defer pc.mu.Unlock() 1559 t.Reused = pc.reused 1560 t.Conn = pc.conn 1561 t.WasIdle = true 1562 if !idleAt.IsZero() { 1563 t.IdleTime = time.Since(idleAt) 1564 } 1565 return 1566 } 1567 1568 func (pc *persistConn) cancelRequest(err error) { 1569 pc.mu.Lock() 1570 defer pc.mu.Unlock() 1571 pc.canceledErr = err 1572 pc.closeLocked(errRequestCanceled) 1573 } 1574 1575 // closeConnIfStillIdle closes the connection if it's still sitting idle. 1576 // This is what's called by the persistConn's idleTimer, and is run in its 1577 // own goroutine. 1578 func (pc *persistConn) closeConnIfStillIdle() { 1579 t := pc.t 1580 t.idleMu.Lock() 1581 defer t.idleMu.Unlock() 1582 if _, ok := t.idleLRU.m[pc]; !ok { 1583 // Not idle. 1584 return 1585 } 1586 t.removeIdleConnLocked(pc) 1587 pc.close(errIdleConnTimeout) 1588 } 1589 1590 // mapRoundTripError returns the appropriate error value for 1591 // persistConn.roundTrip. 1592 // 1593 // The provided err is the first error that (*persistConn).roundTrip 1594 // happened to receive from its select statement. 1595 // 1596 // The startBytesWritten value should be the value of pc.nwrite before the roundTrip 1597 // started writing the request. 1598 func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { 1599 if err == nil { 1600 return nil 1601 } 1602 1603 // If the request was canceled, that's better than network 1604 // failures that were likely the result of tearing down the 1605 // connection. 1606 if cerr := pc.canceled(); cerr != nil { 1607 return cerr 1608 } 1609 1610 // See if an error was set explicitly. 1611 req.mu.Lock() 1612 reqErr := req.err 1613 req.mu.Unlock() 1614 if reqErr != nil { 1615 return reqErr 1616 } 1617 1618 if err == errServerClosedIdle { 1619 // Don't decorate 1620 return err 1621 } 1622 1623 if _, ok := err.(transportReadFromServerError); ok { 1624 // Don't decorate 1625 return err 1626 } 1627 if pc.isBroken() { 1628 <-pc.writeLoopDone 1629 if pc.nwrite == startBytesWritten { 1630 return nothingWrittenError{err} 1631 } 1632 return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) 1633 } 1634 return err 1635 } 1636 1637 // errCallerOwnsConn is an internal sentinel error used when we hand 1638 // off a writable response.Body to the caller. We use this to prevent 1639 // closing a net.Conn that is now owned by the caller. 1640 var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn") 1641 1642 func (pc *persistConn) readLoop() { 1643 closeErr := errReadLoopExiting // default value, if not changed below 1644 defer func() { 1645 pc.close(closeErr) 1646 pc.t.removeIdleConn(pc) 1647 }() 1648 1649 tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { 1650 if err := pc.t.tryPutIdleConn(pc); err != nil { 1651 closeErr = err 1652 if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { 1653 trace.PutIdleConn(err) 1654 } 1655 return false 1656 } 1657 if trace != nil && trace.PutIdleConn != nil { 1658 trace.PutIdleConn(nil) 1659 } 1660 return true 1661 } 1662 1663 // eofc is used to block caller goroutines reading from Response.Body 1664 // at EOF until this goroutines has (potentially) added the connection 1665 // back to the idle pool. 1666 eofc := make(chan struct{}) 1667 defer close(eofc) // unblock reader on errors 1668 1669 // Read this once, before loop starts. (to avoid races in tests) 1670 testHookMu.Lock() 1671 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 1672 testHookMu.Unlock() 1673 1674 alive := true 1675 for alive { 1676 pc.readLimit = pc.maxHeaderResponseSize() 1677 _, err := pc.br.Peek(1) 1678 1679 pc.mu.Lock() 1680 if pc.numExpectedResponses == 0 { 1681 pc.readLoopPeekFailLocked(err) 1682 pc.mu.Unlock() 1683 return 1684 } 1685 pc.mu.Unlock() 1686 1687 rc := <-pc.reqch 1688 trace := httptrace.ContextClientTrace(rc.req.Context()) 1689 1690 var resp *Response 1691 if err == nil { 1692 resp, err = pc.readResponse(rc, trace) 1693 } else { 1694 err = transportReadFromServerError{err} 1695 closeErr = err 1696 } 1697 1698 if err != nil { 1699 if pc.readLimit <= 0 { 1700 err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) 1701 } 1702 1703 select { 1704 case rc.ch <- responseAndError{err: err}: 1705 case <-rc.callerGone: 1706 return 1707 } 1708 return 1709 } 1710 pc.readLimit = maxInt64 // effictively no limit for response bodies 1711 1712 pc.mu.Lock() 1713 pc.numExpectedResponses-- 1714 pc.mu.Unlock() 1715 1716 bodyWritable := resp.bodyIsWritable() 1717 hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 1718 1719 if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable { 1720 // Don't do keep-alive on error if either party requested a close 1721 // or we get an unexpected informational (1xx) response. 1722 // StatusCode 100 is already handled above. 1723 alive = false 1724 } 1725 1726 if !hasBody || bodyWritable { 1727 pc.t.setReqCanceler(rc.req, nil) 1728 1729 // Put the idle conn back into the pool before we send the response 1730 // so if they process it quickly and make another request, they'll 1731 // get this same conn. But we use the unbuffered channel 'rc' 1732 // to guarantee that persistConn.roundTrip got out of its select 1733 // potentially waiting for this persistConn to close. 1734 // but after 1735 alive = alive && 1736 !pc.sawEOF && 1737 pc.wroteRequest() && 1738 tryPutIdleConn(trace) 1739 1740 if bodyWritable { 1741 closeErr = errCallerOwnsConn 1742 } 1743 1744 select { 1745 case rc.ch <- responseAndError{res: resp}: 1746 case <-rc.callerGone: 1747 return 1748 } 1749 1750 // Now that they've read from the unbuffered channel, they're safely 1751 // out of the select that also waits on this goroutine to die, so 1752 // we're allowed to exit now if needed (if alive is false) 1753 testHookReadLoopBeforeNextRead() 1754 continue 1755 } 1756 1757 waitForBodyRead := make(chan bool, 2) 1758 body := &bodyEOFSignal{ 1759 body: resp.Body, 1760 earlyCloseFn: func() error { 1761 waitForBodyRead <- false 1762 <-eofc // will be closed by deferred call at the end of the function 1763 return nil 1764 1765 }, 1766 fn: func(err error) error { 1767 isEOF := err == io.EOF 1768 waitForBodyRead <- isEOF 1769 if isEOF { 1770 <-eofc // see comment above eofc declaration 1771 } else if err != nil { 1772 if cerr := pc.canceled(); cerr != nil { 1773 return cerr 1774 } 1775 } 1776 return err 1777 }, 1778 } 1779 1780 resp.Body = body 1781 if rc.addedGzip && strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { 1782 resp.Body = &gzipReader{body: body} 1783 resp.Header.Del("Content-Encoding") 1784 resp.Header.Del("Content-Length") 1785 resp.ContentLength = -1 1786 resp.Uncompressed = true 1787 } 1788 1789 select { 1790 case rc.ch <- responseAndError{res: resp}: 1791 case <-rc.callerGone: 1792 return 1793 } 1794 1795 // Before looping back to the top of this function and peeking on 1796 // the bufio.Reader, wait for the caller goroutine to finish 1797 // reading the response body. (or for cancelation or death) 1798 select { 1799 case bodyEOF := <-waitForBodyRead: 1800 pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool 1801 alive = alive && 1802 bodyEOF && 1803 !pc.sawEOF && 1804 pc.wroteRequest() && 1805 tryPutIdleConn(trace) 1806 if bodyEOF { 1807 eofc <- struct{}{} 1808 } 1809 case <-rc.req.Cancel: 1810 alive = false 1811 pc.t.CancelRequest(rc.req) 1812 case <-rc.req.Context().Done(): 1813 alive = false 1814 pc.t.cancelRequest(rc.req, rc.req.Context().Err()) 1815 case <-pc.closech: 1816 alive = false 1817 } 1818 1819 testHookReadLoopBeforeNextRead() 1820 } 1821 } 1822 1823 func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 1824 if pc.closed != nil { 1825 return 1826 } 1827 if n := pc.br.Buffered(); n > 0 { 1828 buf, _ := pc.br.Peek(n) 1829 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 1830 } 1831 if peekErr == io.EOF { 1832 // common case. 1833 pc.closeLocked(errServerClosedIdle) 1834 } else { 1835 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr)) 1836 } 1837 } 1838 1839 // readResponse reads an HTTP response (or two, in the case of "Expect: 1840 // 100-continue") from the server. It returns the final non-100 one. 1841 // trace is optional. 1842 func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { 1843 if trace != nil && trace.GotFirstResponseByte != nil { 1844 if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { 1845 trace.GotFirstResponseByte() 1846 } 1847 } 1848 num1xx := 0 // number of informational 1xx headers received 1849 const max1xxResponses = 5 // arbitrary bound on number of informational responses 1850 1851 continueCh := rc.continueCh 1852 for { 1853 resp, err = ReadResponse(pc.br, rc.req) 1854 if err != nil { 1855 return 1856 } 1857 resCode := resp.StatusCode 1858 if continueCh != nil { 1859 if resCode == 100 { 1860 if trace != nil && trace.Got100Continue != nil { 1861 trace.Got100Continue() 1862 } 1863 continueCh <- struct{}{} 1864 continueCh = nil 1865 } else if resCode >= 200 { 1866 close(continueCh) 1867 continueCh = nil 1868 } 1869 } 1870 is1xx := 100 <= resCode && resCode <= 199 1871 // treat 101 as a terminal status, see issue 26161 1872 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols 1873 if is1xxNonTerminal { 1874 num1xx++ 1875 if num1xx > max1xxResponses { 1876 return nil, errors.New("net/http: too many 1xx informational responses") 1877 } 1878 pc.readLimit = pc.maxHeaderResponseSize() // reset the limit 1879 if trace != nil && trace.Got1xxResponse != nil { 1880 if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { 1881 return nil, err 1882 } 1883 } 1884 continue 1885 } 1886 break 1887 } 1888 if resp.isProtocolSwitch() { 1889 resp.Body = newReadWriteCloserBody(pc.br, pc.conn) 1890 } 1891 1892 resp.TLS = pc.tlsState 1893 return 1894 } 1895 1896 // waitForContinue returns the function to block until 1897 // any response, timeout or connection close. After any of them, 1898 // the function returns a bool which indicates if the body should be sent. 1899 func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 1900 if continueCh == nil { 1901 return nil 1902 } 1903 return func() bool { 1904 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 1905 defer timer.Stop() 1906 1907 select { 1908 case _, ok := <-continueCh: 1909 return ok 1910 case <-timer.C: 1911 return true 1912 case <-pc.closech: 1913 return false 1914 } 1915 } 1916 } 1917 1918 func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser { 1919 body := &readWriteCloserBody{ReadWriteCloser: rwc} 1920 if br.Buffered() != 0 { 1921 body.br = br 1922 } 1923 return body 1924 } 1925 1926 // readWriteCloserBody is the Response.Body type used when we want to 1927 // give users write access to the Body through the underlying 1928 // connection (TCP, unless using custom dialers). This is then 1929 // the concrete type for a Response.Body on the 101 Switching 1930 // Protocols response, as used by WebSockets, h2c, etc. 1931 type readWriteCloserBody struct { 1932 br *bufio.Reader // used until empty 1933 io.ReadWriteCloser 1934 } 1935 1936 func (b *readWriteCloserBody) Read(p []byte) (n int, err error) { 1937 if b.br != nil { 1938 if n := b.br.Buffered(); len(p) > n { 1939 p = p[:n] 1940 } 1941 n, err = b.br.Read(p) 1942 if b.br.Buffered() == 0 { 1943 b.br = nil 1944 } 1945 return n, err 1946 } 1947 return b.ReadWriteCloser.Read(p) 1948 } 1949 1950 // nothingWrittenError wraps a write errors which ended up writing zero bytes. 1951 type nothingWrittenError struct { 1952 error 1953 } 1954 1955 func (pc *persistConn) writeLoop() { 1956 defer close(pc.writeLoopDone) 1957 for { 1958 select { 1959 case wr := <-pc.writech: 1960 startBytesWritten := pc.nwrite 1961 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 1962 if bre, ok := err.(requestBodyReadError); ok { 1963 err = bre.error 1964 // Errors reading from the user's 1965 // Request.Body are high priority. 1966 // Set it here before sending on the 1967 // channels below or calling 1968 // pc.close() which tears town 1969 // connections and causes other 1970 // errors. 1971 wr.req.setError(err) 1972 } 1973 if err == nil { 1974 err = pc.bw.Flush() 1975 } 1976 if err != nil { 1977 wr.req.Request.closeBody() 1978 if pc.nwrite == startBytesWritten { 1979 err = nothingWrittenError{err} 1980 } 1981 } 1982 pc.writeErrCh <- err // to the body reader, which might recycle us 1983 wr.ch <- err // to the roundTrip function 1984 if err != nil { 1985 pc.close(err) 1986 return 1987 } 1988 case <-pc.closech: 1989 return 1990 } 1991 } 1992 } 1993 1994 // maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip 1995 // will wait to see the Request's Body.Write result after getting a 1996 // response from the server. See comments in (*persistConn).wroteRequest. 1997 const maxWriteWaitBeforeConnReuse = 50 * time.Millisecond 1998 1999 // wroteRequest is a check before recycling a connection that the previous write 2000 // (from writeLoop above) happened and was successful. 2001 func (pc *persistConn) wroteRequest() bool { 2002 select { 2003 case err := <-pc.writeErrCh: 2004 // Common case: the write happened well before the response, so 2005 // avoid creating a timer. 2006 return err == nil 2007 default: 2008 // Rare case: the request was written in writeLoop above but 2009 // before it could send to pc.writeErrCh, the reader read it 2010 // all, processed it, and called us here. In this case, give the 2011 // write goroutine a bit of time to finish its send. 2012 // 2013 // Less rare case: We also get here in the legitimate case of 2014 // Issue 7569, where the writer is still writing (or stalled), 2015 // but the server has already replied. In this case, we don't 2016 // want to wait too long, and we want to return false so this 2017 // connection isn't re-used. 2018 select { 2019 case err := <-pc.writeErrCh: 2020 return err == nil 2021 case <-time.After(maxWriteWaitBeforeConnReuse): 2022 return false 2023 } 2024 } 2025 } 2026 2027 // responseAndError is how the goroutine reading from an HTTP/1 server 2028 // communicates with the goroutine doing the RoundTrip. 2029 type responseAndError struct { 2030 res *Response // else use this response (see res method) 2031 err error 2032 } 2033 2034 type requestAndChan struct { 2035 req *Request 2036 ch chan responseAndError // unbuffered; always send in select on callerGone 2037 2038 // whether the Transport (as opposed to the user client code) 2039 // added the Accept-Encoding gzip header. If the Transport 2040 // set it, only then do we transparently decode the gzip. 2041 addedGzip bool 2042 2043 // Optional blocking chan for Expect: 100-continue (for send). 2044 // If the request has an "Expect: 100-continue" header and 2045 // the server responds 100 Continue, readLoop send a value 2046 // to writeLoop via this chan. 2047 continueCh chan<- struct{} 2048 2049 callerGone <-chan struct{} // closed when roundTrip caller has returned 2050 } 2051 2052 // A writeRequest is sent by the readLoop's goroutine to the 2053 // writeLoop's goroutine to write a request while the read loop 2054 // concurrently waits on both the write response and the server's 2055 // reply. 2056 type writeRequest struct { 2057 req *transportRequest 2058 ch chan<- error 2059 2060 // Optional blocking chan for Expect: 100-continue (for receive). 2061 // If not nil, writeLoop blocks sending request body until 2062 // it receives from this chan. 2063 continueCh <-chan struct{} 2064 } 2065 2066 type httpError struct { 2067 err string 2068 timeout bool 2069 } 2070 2071 func (e *httpError) Error() string { return e.err } 2072 func (e *httpError) Timeout() bool { return e.timeout } 2073 func (e *httpError) Temporary() bool { return true } 2074 2075 var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} 2076 var errRequestCanceled = errors.New("net/http: request canceled") 2077 var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? 2078 2079 func nop() {} 2080 2081 // testHooks. Always non-nil. 2082 var ( 2083 testHookEnterRoundTrip = nop 2084 testHookWaitResLoop = nop 2085 testHookRoundTripRetried = nop 2086 testHookPrePendingDial = nop 2087 testHookPostPendingDial = nop 2088 2089 testHookMu sync.Locker = fakeLocker{} // guards following 2090 testHookReadLoopBeforeNextRead = nop 2091 ) 2092 2093 func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 2094 testHookEnterRoundTrip() 2095 if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { 2096 pc.t.putOrCloseIdleConn(pc) 2097 return nil, errRequestCanceled 2098 } 2099 pc.mu.Lock() 2100 pc.numExpectedResponses++ 2101 headerFn := pc.mutateHeaderFunc 2102 pc.mu.Unlock() 2103 2104 if headerFn != nil { 2105 headerFn(req.extraHeaders()) 2106 } 2107 2108 // Ask for a compressed version if the caller didn't set their 2109 // own value for Accept-Encoding. We only attempt to 2110 // uncompress the gzip stream if we were the layer that 2111 // requested it. 2112 requestedGzip := false 2113 if !pc.t.DisableCompression && 2114 req.Header.Get("Accept-Encoding") == "" && 2115 req.Header.Get("Range") == "" && 2116 req.Method != "HEAD" { 2117 // Request gzip only, not deflate. Deflate is ambiguous and 2118 // not as universally supported anyway. 2119 // See: https://zlib.net/zlib_faq.html#faq39 2120 // 2121 // Note that we don't request this for HEAD requests, 2122 // due to a bug in nginx: 2123 // https://trac.nginx.org/nginx/ticket/358 2124 // https://golang.org/issue/5522 2125 // 2126 // We don't request gzip if the request is for a range, since 2127 // auto-decoding a portion of a gzipped document will just fail 2128 // anyway. See https://golang.org/issue/8923 2129 requestedGzip = true 2130 req.extraHeaders().Set("Accept-Encoding", "gzip") 2131 } 2132 2133 var continueCh chan struct{} 2134 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 2135 continueCh = make(chan struct{}, 1) 2136 } 2137 2138 if pc.t.DisableKeepAlives && !req.wantsClose() { 2139 req.extraHeaders().Set("Connection", "close") 2140 } 2141 2142 gone := make(chan struct{}) 2143 defer close(gone) 2144 2145 defer func() { 2146 if err != nil { 2147 pc.t.setReqCanceler(req.Request, nil) 2148 } 2149 }() 2150 2151 const debugRoundTrip = false 2152 2153 // Write the request concurrently with waiting for a response, 2154 // in case the server decides to reply before reading our full 2155 // request body. 2156 startBytesWritten := pc.nwrite 2157 writeErrCh := make(chan error, 1) 2158 pc.writech <- writeRequest{req, writeErrCh, continueCh} 2159 2160 resc := make(chan responseAndError) 2161 pc.reqch <- requestAndChan{ 2162 req: req.Request, 2163 ch: resc, 2164 addedGzip: requestedGzip, 2165 continueCh: continueCh, 2166 callerGone: gone, 2167 } 2168 2169 var respHeaderTimer <-chan time.Time 2170 cancelChan := req.Request.Cancel 2171 ctxDoneChan := req.Context().Done() 2172 for { 2173 testHookWaitResLoop() 2174 select { 2175 case err := <-writeErrCh: 2176 if debugRoundTrip { 2177 req.logf("writeErrCh resv: %T/%#v", err, err) 2178 } 2179 if err != nil { 2180 pc.close(fmt.Errorf("write error: %v", err)) 2181 return nil, pc.mapRoundTripError(req, startBytesWritten, err) 2182 } 2183 if d := pc.t.ResponseHeaderTimeout; d > 0 { 2184 if debugRoundTrip { 2185 req.logf("starting timer for %v", d) 2186 } 2187 timer := time.NewTimer(d) 2188 defer timer.Stop() // prevent leaks 2189 respHeaderTimer = timer.C 2190 } 2191 case <-pc.closech: 2192 if debugRoundTrip { 2193 req.logf("closech recv: %T %#v", pc.closed, pc.closed) 2194 } 2195 return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) 2196 case <-respHeaderTimer: 2197 if debugRoundTrip { 2198 req.logf("timeout waiting for response headers.") 2199 } 2200 pc.close(errTimeout) 2201 return nil, errTimeout 2202 case re := <-resc: 2203 if (re.res == nil) == (re.err == nil) { 2204 panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) 2205 } 2206 if debugRoundTrip { 2207 req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) 2208 } 2209 if re.err != nil { 2210 return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) 2211 } 2212 return re.res, nil 2213 case <-cancelChan: 2214 pc.t.CancelRequest(req.Request) 2215 cancelChan = nil 2216 case <-ctxDoneChan: 2217 pc.t.cancelRequest(req.Request, req.Context().Err()) 2218 cancelChan = nil 2219 ctxDoneChan = nil 2220 } 2221 } 2222 } 2223 2224 // tLogKey is a context WithValue key for test debugging contexts containing 2225 // a t.Logf func. See export_test.go's Request.WithT method. 2226 type tLogKey struct{} 2227 2228 func (tr *transportRequest) logf(format string, args ...interface{}) { 2229 if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...interface{})); ok { 2230 logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) 2231 } 2232 } 2233 2234 // markReused marks this connection as having been successfully used for a 2235 // request and response. 2236 func (pc *persistConn) markReused() { 2237 pc.mu.Lock() 2238 pc.reused = true 2239 pc.mu.Unlock() 2240 } 2241 2242 // close closes the underlying TCP connection and closes 2243 // the pc.closech channel. 2244 // 2245 // The provided err is only for testing and debugging; in normal 2246 // circumstances it should never be seen by users. 2247 func (pc *persistConn) close(err error) { 2248 pc.mu.Lock() 2249 defer pc.mu.Unlock() 2250 pc.closeLocked(err) 2251 } 2252 2253 func (pc *persistConn) closeLocked(err error) { 2254 if err == nil { 2255 panic("nil error") 2256 } 2257 pc.broken = true 2258 if pc.closed == nil { 2259 pc.closed = err 2260 if pc.alt != nil { 2261 // Do nothing; can only get here via getConn's 2262 // handlePendingDial's putOrCloseIdleConn when 2263 // it turns out the abandoned connection in 2264 // flight ended up negotiating an alternate 2265 // protocol. We don't use the connection 2266 // freelist for http2. That's done by the 2267 // alternate protocol's RoundTripper. 2268 } else { 2269 if err != errCallerOwnsConn { 2270 pc.conn.Close() 2271 } 2272 close(pc.closech) 2273 } 2274 } 2275 pc.mutateHeaderFunc = nil 2276 } 2277 2278 var portMap = map[string]string{ 2279 "http": "80", 2280 "https": "443", 2281 "socks5": "1080", 2282 } 2283 2284 // canonicalAddr returns url.Host but always with a ":port" suffix 2285 func canonicalAddr(url *url.URL) string { 2286 addr := url.Hostname() 2287 if v, err := idnaASCII(addr); err == nil { 2288 addr = v 2289 } 2290 port := url.Port() 2291 if port == "" { 2292 port = portMap[url.Scheme] 2293 } 2294 return net.JoinHostPort(addr, port) 2295 } 2296 2297 // bodyEOFSignal is used by the HTTP/1 transport when reading response 2298 // bodies to make sure we see the end of a response body before 2299 // proceeding and reading on the connection again. 2300 // 2301 // It wraps a ReadCloser but runs fn (if non-nil) at most 2302 // once, right before its final (error-producing) Read or Close call 2303 // returns. fn should return the new error to return from Read or Close. 2304 // 2305 // If earlyCloseFn is non-nil and Close is called before io.EOF is 2306 // seen, earlyCloseFn is called instead of fn, and its return value is 2307 // the return value from Close. 2308 type bodyEOFSignal struct { 2309 body io.ReadCloser 2310 mu sync.Mutex // guards following 4 fields 2311 closed bool // whether Close has been called 2312 rerr error // sticky Read error 2313 fn func(error) error // err will be nil on Read io.EOF 2314 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 2315 } 2316 2317 var errReadOnClosedResBody = errors.New("http: read on closed response body") 2318 2319 func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 2320 es.mu.Lock() 2321 closed, rerr := es.closed, es.rerr 2322 es.mu.Unlock() 2323 if closed { 2324 return 0, errReadOnClosedResBody 2325 } 2326 if rerr != nil { 2327 return 0, rerr 2328 } 2329 2330 n, err = es.body.Read(p) 2331 if err != nil { 2332 es.mu.Lock() 2333 defer es.mu.Unlock() 2334 if es.rerr == nil { 2335 es.rerr = err 2336 } 2337 err = es.condfn(err) 2338 } 2339 return 2340 } 2341 2342 func (es *bodyEOFSignal) Close() error { 2343 es.mu.Lock() 2344 defer es.mu.Unlock() 2345 if es.closed { 2346 return nil 2347 } 2348 es.closed = true 2349 if es.earlyCloseFn != nil && es.rerr != io.EOF { 2350 return es.earlyCloseFn() 2351 } 2352 err := es.body.Close() 2353 return es.condfn(err) 2354 } 2355 2356 // caller must hold es.mu. 2357 func (es *bodyEOFSignal) condfn(err error) error { 2358 if es.fn == nil { 2359 return err 2360 } 2361 err = es.fn(err) 2362 es.fn = nil 2363 return err 2364 } 2365 2366 // gzipReader wraps a response body so it can lazily 2367 // call gzip.NewReader on the first call to Read 2368 type gzipReader struct { 2369 body *bodyEOFSignal // underlying HTTP/1 response body framing 2370 zr *gzip.Reader // lazily-initialized gzip reader 2371 zerr error // any error from gzip.NewReader; sticky 2372 } 2373 2374 func (gz *gzipReader) Read(p []byte) (n int, err error) { 2375 if gz.zr == nil { 2376 if gz.zerr == nil { 2377 gz.zr, gz.zerr = gzip.NewReader(gz.body) 2378 } 2379 if gz.zerr != nil { 2380 return 0, gz.zerr 2381 } 2382 } 2383 2384 gz.body.mu.Lock() 2385 if gz.body.closed { 2386 err = errReadOnClosedResBody 2387 } 2388 gz.body.mu.Unlock() 2389 2390 if err != nil { 2391 return 0, err 2392 } 2393 return gz.zr.Read(p) 2394 } 2395 2396 func (gz *gzipReader) Close() error { 2397 return gz.body.Close() 2398 } 2399 2400 type readerAndCloser struct { 2401 io.Reader 2402 io.Closer 2403 } 2404 2405 type tlsHandshakeTimeoutError struct{} 2406 2407 func (tlsHandshakeTimeoutError) Timeout() bool { return true } 2408 func (tlsHandshakeTimeoutError) Temporary() bool { return true } 2409 func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 2410 2411 // fakeLocker is a sync.Locker which does nothing. It's used to guard 2412 // test-only fields when not under test, to avoid runtime atomic 2413 // overhead. 2414 type fakeLocker struct{} 2415 2416 func (fakeLocker) Lock() {} 2417 func (fakeLocker) Unlock() {} 2418 2419 // cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if 2420 // cfg is nil. This is safe to call even if cfg is in active use by a TLS 2421 // client or server. 2422 func cloneTLSConfig(cfg *tls.Config) *tls.Config { 2423 if cfg == nil { 2424 return &tls.Config{} 2425 } 2426 return cfg.Clone() 2427 } 2428 2429 type connLRU struct { 2430 ll *list.List // list.Element.Value type is of *persistConn 2431 m map[*persistConn]*list.Element 2432 } 2433 2434 // add adds pc to the head of the linked list. 2435 func (cl *connLRU) add(pc *persistConn) { 2436 if cl.ll == nil { 2437 cl.ll = list.New() 2438 cl.m = make(map[*persistConn]*list.Element) 2439 } 2440 ele := cl.ll.PushFront(pc) 2441 if _, ok := cl.m[pc]; ok { 2442 panic("persistConn was already in LRU") 2443 } 2444 cl.m[pc] = ele 2445 } 2446 2447 func (cl *connLRU) removeOldest() *persistConn { 2448 ele := cl.ll.Back() 2449 pc := ele.Value.(*persistConn) 2450 cl.ll.Remove(ele) 2451 delete(cl.m, pc) 2452 return pc 2453 } 2454 2455 // remove removes pc from cl. 2456 func (cl *connLRU) remove(pc *persistConn) { 2457 if ele, ok := cl.m[pc]; ok { 2458 cl.ll.Remove(ele) 2459 delete(cl.m, pc) 2460 } 2461 } 2462 2463 // len returns the number of items in the cache. 2464 func (cl *connLRU) len() int { 2465 return len(cl.m) 2466 } 2467 2468 // validPort reports whether p (without the colon) is a valid port in 2469 // a URL, per RFC 3986 Section 3.2.3, which says the port may be 2470 // empty, or only contain digits. 2471 func validPort(p string) bool { 2472 for _, r := range []byte(p) { 2473 if r < '0' || r > '9' { 2474 return false 2475 } 2476 } 2477 return true 2478 }