github.com/goproxy0/go@v0.0.0-20171111080102-49cc0c489d2c/src/net/http/transport.go (about)

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // HTTP client implementation. See RFC 2616.
     6  //
     7  // This is the low-level Transport implementation of RoundTripper.
     8  // The high-level interface is in client.go.
     9  
    10  package http
    11  
    12  import (
    13  	"bufio"
    14  	"compress/gzip"
    15  	"container/list"
    16  	"context"
    17  	"crypto/tls"
    18  	"errors"
    19  	"fmt"
    20  	"io"
    21  	"log"
    22  	"net"
    23  	"net/http/httptrace"
    24  	"net/url"
    25  	"os"
    26  	"strings"
    27  	"sync"
    28  	"sync/atomic"
    29  	"time"
    30  
    31  	"golang_org/x/net/lex/httplex"
    32  	"golang_org/x/net/proxy"
    33  )
    34  
    35  // DefaultTransport is the default implementation of Transport and is
    36  // used by DefaultClient. It establishes network connections as needed
    37  // and caches them for reuse by subsequent calls. It uses HTTP proxies
    38  // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
    39  // $no_proxy) environment variables.
    40  var DefaultTransport RoundTripper = &Transport{
    41  	Proxy: ProxyFromEnvironment,
    42  	DialContext: (&net.Dialer{
    43  		Timeout:   30 * time.Second,
    44  		KeepAlive: 30 * time.Second,
    45  		DualStack: true,
    46  	}).DialContext,
    47  	MaxIdleConns:          100,
    48  	IdleConnTimeout:       90 * time.Second,
    49  	TLSHandshakeTimeout:   10 * time.Second,
    50  	ExpectContinueTimeout: 1 * time.Second,
    51  }
    52  
    53  // DefaultMaxIdleConnsPerHost is the default value of Transport's
    54  // MaxIdleConnsPerHost.
    55  const DefaultMaxIdleConnsPerHost = 2
    56  
    57  // Transport is an implementation of RoundTripper that supports HTTP,
    58  // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
    59  //
    60  // By default, Transport caches connections for future re-use.
    61  // This may leave many open connections when accessing many hosts.
    62  // This behavior can be managed using Transport's CloseIdleConnections method
    63  // and the MaxIdleConnsPerHost and DisableKeepAlives fields.
    64  //
    65  // Transports should be reused instead of created as needed.
    66  // Transports are safe for concurrent use by multiple goroutines.
    67  //
    68  // A Transport is a low-level primitive for making HTTP and HTTPS requests.
    69  // For high-level functionality, such as cookies and redirects, see Client.
    70  //
    71  // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
    72  // for HTTPS URLs, depending on whether the server supports HTTP/2,
    73  // and how the Transport is configured. The DefaultTransport supports HTTP/2.
    74  // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2
    75  // and call ConfigureTransport. See the package docs for more about HTTP/2.
    76  type Transport struct {
    77  	idleMu     sync.Mutex
    78  	wantIdle   bool                                // user has requested to close all idle conns
    79  	idleConn   map[connectMethodKey][]*persistConn // most recently used at end
    80  	idleConnCh map[connectMethodKey]chan *persistConn
    81  	idleLRU    connLRU
    82  
    83  	reqMu       sync.Mutex
    84  	reqCanceler map[*Request]func(error)
    85  
    86  	altMu    sync.Mutex   // guards changing altProto only
    87  	altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme
    88  
    89  	// Proxy specifies a function to return a proxy for a given
    90  	// Request. If the function returns a non-nil error, the
    91  	// request is aborted with the provided error.
    92  	//
    93  	// The proxy type is determined by the URL scheme. "http"
    94  	// and "socks5" are supported. If the scheme is empty,
    95  	// "http" is assumed.
    96  	//
    97  	// If Proxy is nil or returns a nil *URL, no proxy is used.
    98  	Proxy func(*Request) (*url.URL, error)
    99  
   100  	// GetConnectMethodAddr specifies a function to return a addr string for ConnectMethod
   101  	GetConnectMethodAddr func(addr string) string
   102  
   103  	// DialContext specifies the dial function for creating unencrypted TCP connections.
   104  	// If DialContext is nil (and the deprecated Dial below is also nil),
   105  	// then the transport dials using package net.
   106  	DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
   107  
   108  	// Dial specifies the dial function for creating unencrypted TCP connections.
   109  	//
   110  	// Deprecated: Use DialContext instead, which allows the transport
   111  	// to cancel dials as soon as they are no longer needed.
   112  	// If both are set, DialContext takes priority.
   113  	Dial func(network, addr string) (net.Conn, error)
   114  
   115  	// DialTLS specifies an optional dial function for creating
   116  	// TLS connections for non-proxied HTTPS requests.
   117  	//
   118  	// If DialTLS is nil, Dial and TLSClientConfig are used.
   119  	//
   120  	// If DialTLS is set, the Dial hook is not used for HTTPS
   121  	// requests and the TLSClientConfig and TLSHandshakeTimeout
   122  	// are ignored. The returned net.Conn is assumed to already be
   123  	// past the TLS handshake.
   124  	DialTLS func(network, addr string) (net.Conn, error)
   125  
   126  	// TLSClientConfig specifies the TLS configuration to use with
   127  	// tls.Client.
   128  	// If nil, the default configuration is used.
   129  	// If non-nil, HTTP/2 support may not be enabled by default.
   130  	TLSClientConfig *tls.Config
   131  
   132  	// TLSHandshakeTimeout specifies the maximum amount of time waiting to
   133  	// wait for a TLS handshake. Zero means no timeout.
   134  	TLSHandshakeTimeout time.Duration
   135  
   136  	// DisableKeepAlives, if true, prevents re-use of TCP connections
   137  	// between different HTTP requests.
   138  	DisableKeepAlives bool
   139  
   140  	// DisableCompression, if true, prevents the Transport from
   141  	// requesting compression with an "Accept-Encoding: gzip"
   142  	// request header when the Request contains no existing
   143  	// Accept-Encoding value. If the Transport requests gzip on
   144  	// its own and gets a gzipped response, it's transparently
   145  	// decoded in the Response.Body. However, if the user
   146  	// explicitly requested gzip it is not automatically
   147  	// uncompressed.
   148  	DisableCompression bool
   149  
   150  	// MaxIdleConns controls the maximum number of idle (keep-alive)
   151  	// connections across all hosts. Zero means no limit.
   152  	MaxIdleConns int
   153  
   154  	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
   155  	// (keep-alive) connections to keep per-host. If zero,
   156  	// DefaultMaxIdleConnsPerHost is used.
   157  	MaxIdleConnsPerHost int
   158  
   159  	// IdleConnTimeout is the maximum amount of time an idle
   160  	// (keep-alive) connection will remain idle before closing
   161  	// itself.
   162  	// Zero means no limit.
   163  	IdleConnTimeout time.Duration
   164  
   165  	// ResponseHeaderTimeout, if non-zero, specifies the amount of
   166  	// time to wait for a server's response headers after fully
   167  	// writing the request (including its body, if any). This
   168  	// time does not include the time to read the response body.
   169  	ResponseHeaderTimeout time.Duration
   170  
   171  	// ExpectContinueTimeout, if non-zero, specifies the amount of
   172  	// time to wait for a server's first response headers after fully
   173  	// writing the request headers if the request has an
   174  	// "Expect: 100-continue" header. Zero means no timeout and
   175  	// causes the body to be sent immediately, without
   176  	// waiting for the server to approve.
   177  	// This time does not include the time to send the request header.
   178  	ExpectContinueTimeout time.Duration
   179  
   180  	// TLSNextProto specifies how the Transport switches to an
   181  	// alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
   182  	// protocol negotiation. If Transport dials an TLS connection
   183  	// with a non-empty protocol name and TLSNextProto contains a
   184  	// map entry for that key (such as "h2"), then the func is
   185  	// called with the request's authority (such as "example.com"
   186  	// or "example.com:1234") and the TLS connection. The function
   187  	// must return a RoundTripper that then handles the request.
   188  	// If TLSNextProto is not nil, HTTP/2 support is not enabled
   189  	// automatically.
   190  	TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper
   191  
   192  	// ProxyConnectHeader optionally specifies headers to send to
   193  	// proxies during CONNECT requests.
   194  	ProxyConnectHeader Header
   195  
   196  	// MaxResponseHeaderBytes specifies a limit on how many
   197  	// response bytes are allowed in the server's response
   198  	// header.
   199  	//
   200  	// Zero means to use a default limit.
   201  	MaxResponseHeaderBytes int64
   202  
   203  	// nextProtoOnce guards initialization of TLSNextProto and
   204  	// h2transport (via onceSetNextProtoDefaults)
   205  	nextProtoOnce sync.Once
   206  	h2transport   *http2Transport // non-nil if http2 wired up
   207  
   208  	// TODO: tunable on max per-host TCP dials in flight (Issue 13957)
   209  }
   210  
   211  // onceSetNextProtoDefaults initializes TLSNextProto.
   212  // It must be called via t.nextProtoOnce.Do.
   213  func (t *Transport) onceSetNextProtoDefaults() {
   214  	if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") {
   215  		return
   216  	}
   217  	if t.TLSNextProto != nil {
   218  		// This is the documented way to disable http2 on a
   219  		// Transport.
   220  		return
   221  	}
   222  	if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil {
   223  		// Be conservative and don't automatically enable
   224  		// http2 if they've specified a custom TLS config or
   225  		// custom dialers. Let them opt-in themselves via
   226  		// http2.ConfigureTransport so we don't surprise them
   227  		// by modifying their tls.Config. Issue 14275.
   228  		return
   229  	}
   230  	t2, err := http2configureTransport(t)
   231  	if err != nil {
   232  		log.Printf("Error enabling Transport HTTP/2 support: %v", err)
   233  		return
   234  	}
   235  	t.h2transport = t2
   236  
   237  	// Auto-configure the http2.Transport's MaxHeaderListSize from
   238  	// the http.Transport's MaxResponseHeaderBytes. They don't
   239  	// exactly mean the same thing, but they're close.
   240  	//
   241  	// TODO: also add this to x/net/http2.Configure Transport, behind
   242  	// a +build go1.7 build tag:
   243  	if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 {
   244  		const h2max = 1<<32 - 1
   245  		if limit1 >= h2max {
   246  			t2.MaxHeaderListSize = h2max
   247  		} else {
   248  			t2.MaxHeaderListSize = uint32(limit1)
   249  		}
   250  	}
   251  }
   252  
   253  // ProxyFromEnvironment returns the URL of the proxy to use for a
   254  // given request, as indicated by the environment variables
   255  // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
   256  // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
   257  // requests.
   258  //
   259  // The environment values may be either a complete URL or a
   260  // "host[:port]", in which case the "http" scheme is assumed.
   261  // An error is returned if the value is a different form.
   262  //
   263  // A nil URL and nil error are returned if no proxy is defined in the
   264  // environment, or a proxy should not be used for the given request,
   265  // as defined by NO_PROXY.
   266  //
   267  // As a special case, if req.URL.Host is "localhost" (with or without
   268  // a port number), then a nil URL and nil error will be returned.
   269  func ProxyFromEnvironment(req *Request) (*url.URL, error) {
   270  	var proxy string
   271  	if req.URL.Scheme == "https" {
   272  		proxy = httpsProxyEnv.Get()
   273  	}
   274  	if proxy == "" {
   275  		proxy = httpProxyEnv.Get()
   276  		if proxy != "" && os.Getenv("REQUEST_METHOD") != "" {
   277  			return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")
   278  		}
   279  	}
   280  	if proxy == "" {
   281  		return nil, nil
   282  	}
   283  	if !useProxy(canonicalAddr(req.URL)) {
   284  		return nil, nil
   285  	}
   286  	proxyURL, err := url.Parse(proxy)
   287  	if err != nil ||
   288  		(proxyURL.Scheme != "http" &&
   289  			proxyURL.Scheme != "https" &&
   290  			proxyURL.Scheme != "socks5") {
   291  		// proxy was bogus. Try prepending "http://" to it and
   292  		// see if that parses correctly. If not, we fall
   293  		// through and complain about the original one.
   294  		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
   295  			return proxyURL, nil
   296  		}
   297  
   298  	}
   299  	if err != nil {
   300  		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
   301  	}
   302  	return proxyURL, nil
   303  }
   304  
   305  // ProxyURL returns a proxy function (for use in a Transport)
   306  // that always returns the same URL.
   307  func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
   308  	return func(*Request) (*url.URL, error) {
   309  		return fixedURL, nil
   310  	}
   311  }
   312  
   313  // transportRequest is a wrapper around a *Request that adds
   314  // optional extra headers to write and stores any error to return
   315  // from roundTrip.
   316  type transportRequest struct {
   317  	*Request                        // original request, not to be mutated
   318  	extra    Header                 // extra headers to write, or nil
   319  	trace    *httptrace.ClientTrace // optional
   320  
   321  	mu  sync.Mutex // guards err
   322  	err error      // first setError value for mapRoundTripError to consider
   323  }
   324  
   325  func (tr *transportRequest) extraHeaders() Header {
   326  	if tr.extra == nil {
   327  		tr.extra = make(Header)
   328  	}
   329  	return tr.extra
   330  }
   331  
   332  func (tr *transportRequest) setError(err error) {
   333  	tr.mu.Lock()
   334  	if tr.err == nil {
   335  		tr.err = err
   336  	}
   337  	tr.mu.Unlock()
   338  }
   339  
   340  // RoundTrip implements the RoundTripper interface.
   341  //
   342  // For higher-level HTTP client support (such as handling of cookies
   343  // and redirects), see Get, Post, and the Client type.
   344  func (t *Transport) RoundTrip(req *Request) (*Response, error) {
   345  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   346  	ctx := req.Context()
   347  	trace := httptrace.ContextClientTrace(ctx)
   348  
   349  	if req.URL == nil {
   350  		req.closeBody()
   351  		return nil, errors.New("http: nil Request.URL")
   352  	}
   353  	if req.Header == nil {
   354  		req.closeBody()
   355  		return nil, errors.New("http: nil Request.Header")
   356  	}
   357  	scheme := req.URL.Scheme
   358  	isHTTP := scheme == "http" || scheme == "https"
   359  	if isHTTP {
   360  		for k, vv := range req.Header {
   361  			if !httplex.ValidHeaderFieldName(k) {
   362  				return nil, fmt.Errorf("net/http: invalid header field name %q", k)
   363  			}
   364  			for _, v := range vv {
   365  				if !httplex.ValidHeaderFieldValue(v) {
   366  					return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k)
   367  				}
   368  			}
   369  		}
   370  	}
   371  
   372  	altProto, _ := t.altProto.Load().(map[string]RoundTripper)
   373  	if altRT := altProto[scheme]; altRT != nil {
   374  		if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
   375  			return resp, err
   376  		}
   377  	}
   378  	if !isHTTP {
   379  		req.closeBody()
   380  		return nil, &badStringError{"unsupported protocol scheme", scheme}
   381  	}
   382  	if req.Method != "" && !validMethod(req.Method) {
   383  		return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
   384  	}
   385  	if req.URL.Host == "" {
   386  		req.closeBody()
   387  		return nil, errors.New("http: no Host in request URL")
   388  	}
   389  
   390  	for {
   391  		// treq gets modified by roundTrip, so we need to recreate for each retry.
   392  		treq := &transportRequest{Request: req, trace: trace}
   393  		cm, err := t.connectMethodForRequest(treq)
   394  		if err != nil {
   395  			req.closeBody()
   396  			return nil, err
   397  		}
   398  
   399  		// Get the cached or newly-created connection to either the
   400  		// host (for http or https), the http proxy, or the http proxy
   401  		// pre-CONNECTed to https server. In any case, we'll be ready
   402  		// to send it requests.
   403  		pconn, err := t.getConn(treq, cm)
   404  		if err != nil {
   405  			t.setReqCanceler(req, nil)
   406  			req.closeBody()
   407  			return nil, err
   408  		}
   409  
   410  		var resp *Response
   411  		if pconn.alt != nil {
   412  			// HTTP/2 path.
   413  			t.setReqCanceler(req, nil) // not cancelable with CancelRequest
   414  			resp, err = pconn.alt.RoundTrip(req)
   415  		} else {
   416  			resp, err = pconn.roundTrip(treq)
   417  		}
   418  		if err == nil {
   419  			return resp, nil
   420  		}
   421  		if !pconn.shouldRetryRequest(req, err) {
   422  			// Issue 16465: return underlying net.Conn.Read error from peek,
   423  			// as we've historically done.
   424  			if e, ok := err.(transportReadFromServerError); ok {
   425  				err = e.err
   426  			}
   427  			return nil, err
   428  		}
   429  		testHookRoundTripRetried()
   430  
   431  		// Rewind the body if we're able to.  (HTTP/2 does this itself so we only
   432  		// need to do it for HTTP/1.1 connections.)
   433  		if req.GetBody != nil && pconn.alt == nil {
   434  			newReq := *req
   435  			var err error
   436  			newReq.Body, err = req.GetBody()
   437  			if err != nil {
   438  				return nil, err
   439  			}
   440  			req = &newReq
   441  		}
   442  	}
   443  }
   444  
   445  // shouldRetryRequest reports whether we should retry sending a failed
   446  // HTTP request on a new connection. The non-nil input error is the
   447  // error from roundTrip.
   448  func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool {
   449  	if err == http2ErrNoCachedConn {
   450  		// Issue 16582: if the user started a bunch of
   451  		// requests at once, they can all pick the same conn
   452  		// and violate the server's max concurrent streams.
   453  		// Instead, match the HTTP/1 behavior for now and dial
   454  		// again to get a new TCP connection, rather than failing
   455  		// this request.
   456  		return true
   457  	}
   458  	if err == errMissingHost {
   459  		// User error.
   460  		return false
   461  	}
   462  	if !pc.isReused() {
   463  		// This was a fresh connection. There's no reason the server
   464  		// should've hung up on us.
   465  		//
   466  		// Also, if we retried now, we could loop forever
   467  		// creating new connections and retrying if the server
   468  		// is just hanging up on us because it doesn't like
   469  		// our request (as opposed to sending an error).
   470  		return false
   471  	}
   472  	if _, ok := err.(nothingWrittenError); ok {
   473  		// We never wrote anything, so it's safe to retry, if there's no body or we
   474  		// can "rewind" the body with GetBody.
   475  		return req.outgoingLength() == 0 || req.GetBody != nil
   476  	}
   477  	if !req.isReplayable() {
   478  		// Don't retry non-idempotent requests.
   479  		return false
   480  	}
   481  	if _, ok := err.(transportReadFromServerError); ok {
   482  		// We got some non-EOF net.Conn.Read failure reading
   483  		// the 1st response byte from the server.
   484  		return true
   485  	}
   486  	if err == errServerClosedIdle {
   487  		// The server replied with io.EOF while we were trying to
   488  		// read the response. Probably an unfortunately keep-alive
   489  		// timeout, just as the client was writing a request.
   490  		return true
   491  	}
   492  	return false // conservatively
   493  }
   494  
   495  // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
   496  var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")
   497  
   498  // RegisterProtocol registers a new protocol with scheme.
   499  // The Transport will pass requests using the given scheme to rt.
   500  // It is rt's responsibility to simulate HTTP request semantics.
   501  //
   502  // RegisterProtocol can be used by other packages to provide
   503  // implementations of protocol schemes like "ftp" or "file".
   504  //
   505  // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
   506  // handle the RoundTrip itself for that one request, as if the
   507  // protocol were not registered.
   508  func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
   509  	t.altMu.Lock()
   510  	defer t.altMu.Unlock()
   511  	oldMap, _ := t.altProto.Load().(map[string]RoundTripper)
   512  	if _, exists := oldMap[scheme]; exists {
   513  		panic("protocol " + scheme + " already registered")
   514  	}
   515  	newMap := make(map[string]RoundTripper)
   516  	for k, v := range oldMap {
   517  		newMap[k] = v
   518  	}
   519  	newMap[scheme] = rt
   520  	t.altProto.Store(newMap)
   521  }
   522  
   523  // CloseIdleConnections closes any connections which were previously
   524  // connected from previous requests but are now sitting idle in
   525  // a "keep-alive" state. It does not interrupt any connections currently
   526  // in use.
   527  func (t *Transport) CloseIdleConnections() {
   528  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   529  	t.idleMu.Lock()
   530  	m := t.idleConn
   531  	t.idleConn = nil
   532  	t.idleConnCh = nil
   533  	t.wantIdle = true
   534  	t.idleLRU = connLRU{}
   535  	t.idleMu.Unlock()
   536  	for _, conns := range m {
   537  		for _, pconn := range conns {
   538  			pconn.close(errCloseIdleConns)
   539  		}
   540  	}
   541  	if t2 := t.h2transport; t2 != nil {
   542  		t2.CloseIdleConnections()
   543  	}
   544  }
   545  
   546  var errCloseConn = errors.New("http: CloseConnection called")
   547  
   548  // CloseConnections closes any connections which were previously
   549  // connected from previous requests. It may interrupt connections
   550  // currently in use.
   551  func (t *Transport) CloseConnection(f func(raddr net.Addr) bool) {
   552  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   553  	t.idleMu.Lock()
   554  	m := t.idleConn
   555  	t.idleConn = nil
   556  	t.idleConnCh = nil
   557  	t.wantIdle = true
   558  	t.idleLRU = connLRU{}
   559  	t.idleMu.Unlock()
   560  	for _, conns := range m {
   561  		for _, pconn := range conns {
   562  			if f(pconn.conn.RemoteAddr()) {
   563  				pconn.close(errCloseConn)
   564  			}
   565  		}
   566  	}
   567  	if t2 := t.h2transport; t2 != nil {
   568  		if p, ok := t2.connPool().(*http2clientConnPool); ok {
   569  			p.mu.Lock()
   570  			defer p.mu.Unlock()
   571  
   572  			for _, vv := range p.conns {
   573  				for _, cc := range vv {
   574  					cc.mu.Lock()
   575  					if f(cc.tconn.RemoteAddr()) {
   576  						cc.closed = true
   577  					}
   578  					cc.mu.Unlock()
   579  					if cc.closed {
   580  						cc.tconn.Close()
   581  					}
   582  				}
   583  			}
   584  		}
   585  	}
   586  }
   587  
   588  // CancelRequest cancels an in-flight request by closing its connection.
   589  // CancelRequest should only be called after RoundTrip has returned.
   590  //
   591  // Deprecated: Use Request.WithContext to create a request with a
   592  // cancelable context instead. CancelRequest cannot cancel HTTP/2
   593  // requests.
   594  func (t *Transport) CancelRequest(req *Request) {
   595  	t.cancelRequest(req, errRequestCanceled)
   596  }
   597  
   598  // Cancel an in-flight request, recording the error value.
   599  func (t *Transport) cancelRequest(req *Request, err error) {
   600  	t.reqMu.Lock()
   601  	cancel := t.reqCanceler[req]
   602  	delete(t.reqCanceler, req)
   603  	t.reqMu.Unlock()
   604  	if cancel != nil {
   605  		cancel(err)
   606  	}
   607  }
   608  
   609  //
   610  // Private implementation past this point.
   611  //
   612  
   613  var (
   614  	httpProxyEnv = &envOnce{
   615  		names: []string{"HTTP_PROXY", "http_proxy"},
   616  	}
   617  	httpsProxyEnv = &envOnce{
   618  		names: []string{"HTTPS_PROXY", "https_proxy"},
   619  	}
   620  	noProxyEnv = &envOnce{
   621  		names: []string{"NO_PROXY", "no_proxy"},
   622  	}
   623  )
   624  
   625  // envOnce looks up an environment variable (optionally by multiple
   626  // names) once. It mitigates expensive lookups on some platforms
   627  // (e.g. Windows).
   628  type envOnce struct {
   629  	names []string
   630  	once  sync.Once
   631  	val   string
   632  }
   633  
   634  func (e *envOnce) Get() string {
   635  	e.once.Do(e.init)
   636  	return e.val
   637  }
   638  
   639  func (e *envOnce) init() {
   640  	for _, n := range e.names {
   641  		e.val = os.Getenv(n)
   642  		if e.val != "" {
   643  			return
   644  		}
   645  	}
   646  }
   647  
   648  // reset is used by tests
   649  func (e *envOnce) reset() {
   650  	e.once = sync.Once{}
   651  	e.val = ""
   652  }
   653  
   654  func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
   655  	if port := treq.URL.Port(); !validPort(port) {
   656  		return cm, fmt.Errorf("invalid URL port %q", port)
   657  	}
   658  	cm.GetConnectMethodAddr = t.GetConnectMethodAddr
   659  	cm.targetScheme = treq.URL.Scheme
   660  	cm.targetAddr = canonicalAddr(treq.URL)
   661  	if t.Proxy != nil {
   662  		cm.proxyURL, err = t.Proxy(treq.Request)
   663  		if err == nil && cm.proxyURL != nil {
   664  			if port := cm.proxyURL.Port(); !validPort(port) {
   665  				return cm, fmt.Errorf("invalid proxy URL port %q", port)
   666  			}
   667  		}
   668  	}
   669  	return cm, err
   670  }
   671  
   672  // proxyAuth returns the Proxy-Authorization header to set
   673  // on requests, if applicable.
   674  func (cm *connectMethod) proxyAuth() string {
   675  	if cm.proxyURL == nil {
   676  		return ""
   677  	}
   678  	if u := cm.proxyURL.User; u != nil {
   679  		username := u.Username()
   680  		password, _ := u.Password()
   681  		return "Basic " + basicAuth(username, password)
   682  	}
   683  	return ""
   684  }
   685  
   686  // error values for debugging and testing, not seen by users.
   687  var (
   688  	errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled")
   689  	errConnBroken         = errors.New("http: putIdleConn: connection is in bad state")
   690  	errWantIdle           = errors.New("http: putIdleConn: CloseIdleConnections was called")
   691  	errTooManyIdle        = errors.New("http: putIdleConn: too many idle connections")
   692  	errTooManyIdleHost    = errors.New("http: putIdleConn: too many idle connections for host")
   693  	errCloseIdleConns     = errors.New("http: CloseIdleConnections called")
   694  	errReadLoopExiting    = errors.New("http: persistConn.readLoop exiting")
   695  	errServerClosedIdle   = errors.New("http: server closed idle connection")
   696  	errIdleConnTimeout    = errors.New("http: idle connection timeout")
   697  	errNotCachingH2Conn   = errors.New("http: not caching alternate protocol's connections")
   698  )
   699  
   700  // transportReadFromServerError is used by Transport.readLoop when the
   701  // 1 byte peek read fails and we're actually anticipating a response.
   702  // Usually this is just due to the inherent keep-alive shut down race,
   703  // where the server closed the connection at the same time the client
   704  // wrote. The underlying err field is usually io.EOF or some
   705  // ECONNRESET sort of thing which varies by platform. But it might be
   706  // the user's custom net.Conn.Read error too, so we carry it along for
   707  // them to return from Transport.RoundTrip.
   708  type transportReadFromServerError struct {
   709  	err error
   710  }
   711  
   712  func (e transportReadFromServerError) Error() string {
   713  	return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err)
   714  }
   715  
   716  func (t *Transport) putOrCloseIdleConn(pconn *persistConn) {
   717  	if err := t.tryPutIdleConn(pconn); err != nil {
   718  		pconn.close(err)
   719  	}
   720  }
   721  
   722  func (t *Transport) maxIdleConnsPerHost() int {
   723  	if v := t.MaxIdleConnsPerHost; v != 0 {
   724  		return v
   725  	}
   726  	return DefaultMaxIdleConnsPerHost
   727  }
   728  
   729  // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
   730  // a new request.
   731  // If pconn is no longer needed or not in a good state, tryPutIdleConn returns
   732  // an error explaining why it wasn't registered.
   733  // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
   734  func (t *Transport) tryPutIdleConn(pconn *persistConn) error {
   735  	if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
   736  		return errKeepAlivesDisabled
   737  	}
   738  	if pconn.isBroken() {
   739  		return errConnBroken
   740  	}
   741  	if pconn.alt != nil {
   742  		return errNotCachingH2Conn
   743  	}
   744  	pconn.markReused()
   745  	key := pconn.cacheKey
   746  
   747  	t.idleMu.Lock()
   748  	defer t.idleMu.Unlock()
   749  
   750  	waitingDialer := t.idleConnCh[key]
   751  	select {
   752  	case waitingDialer <- pconn:
   753  		// We're done with this pconn and somebody else is
   754  		// currently waiting for a conn of this type (they're
   755  		// actively dialing, but this conn is ready
   756  		// first). Chrome calls this socket late binding. See
   757  		// https://insouciant.org/tech/connection-management-in-chromium/
   758  		return nil
   759  	default:
   760  		if waitingDialer != nil {
   761  			// They had populated this, but their dial won
   762  			// first, so we can clean up this map entry.
   763  			delete(t.idleConnCh, key)
   764  		}
   765  	}
   766  	if t.wantIdle {
   767  		return errWantIdle
   768  	}
   769  	if t.idleConn == nil {
   770  		t.idleConn = make(map[connectMethodKey][]*persistConn)
   771  	}
   772  	idles := t.idleConn[key]
   773  	if len(idles) >= t.maxIdleConnsPerHost() {
   774  		return errTooManyIdleHost
   775  	}
   776  	for _, exist := range idles {
   777  		if exist == pconn {
   778  			log.Fatalf("dup idle pconn %p in freelist", pconn)
   779  		}
   780  	}
   781  	t.idleConn[key] = append(idles, pconn)
   782  	t.idleLRU.add(pconn)
   783  	if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns {
   784  		oldest := t.idleLRU.removeOldest()
   785  		oldest.close(errTooManyIdle)
   786  		t.removeIdleConnLocked(oldest)
   787  	}
   788  	if t.IdleConnTimeout > 0 {
   789  		if pconn.idleTimer != nil {
   790  			pconn.idleTimer.Reset(t.IdleConnTimeout)
   791  		} else {
   792  			pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle)
   793  		}
   794  	}
   795  	pconn.idleAt = time.Now()
   796  	return nil
   797  }
   798  
   799  // getIdleConnCh returns a channel to receive and return idle
   800  // persistent connection for the given connectMethod.
   801  // It may return nil, if persistent connections are not being used.
   802  func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn {
   803  	if t.DisableKeepAlives {
   804  		return nil
   805  	}
   806  	key := cm.key()
   807  	t.idleMu.Lock()
   808  	defer t.idleMu.Unlock()
   809  	t.wantIdle = false
   810  	if t.idleConnCh == nil {
   811  		t.idleConnCh = make(map[connectMethodKey]chan *persistConn)
   812  	}
   813  	ch, ok := t.idleConnCh[key]
   814  	if !ok {
   815  		ch = make(chan *persistConn)
   816  		t.idleConnCh[key] = ch
   817  	}
   818  	return ch
   819  }
   820  
   821  func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn, idleSince time.Time) {
   822  	key := cm.key()
   823  	t.idleMu.Lock()
   824  	defer t.idleMu.Unlock()
   825  	for {
   826  		pconns, ok := t.idleConn[key]
   827  		if !ok {
   828  			return nil, time.Time{}
   829  		}
   830  		if len(pconns) == 1 {
   831  			pconn = pconns[0]
   832  			delete(t.idleConn, key)
   833  		} else {
   834  			// 2 or more cached connections; use the most
   835  			// recently used one at the end.
   836  			pconn = pconns[len(pconns)-1]
   837  			t.idleConn[key] = pconns[:len(pconns)-1]
   838  		}
   839  		t.idleLRU.remove(pconn)
   840  		if pconn.isBroken() {
   841  			// There is a tiny window where this is
   842  			// possible, between the connecting dying and
   843  			// the persistConn readLoop calling
   844  			// Transport.removeIdleConn. Just skip it and
   845  			// carry on.
   846  			continue
   847  		}
   848  		if pconn.idleTimer != nil && !pconn.idleTimer.Stop() {
   849  			// We picked this conn at the ~same time it
   850  			// was expiring and it's trying to close
   851  			// itself in another goroutine. Don't use it.
   852  			continue
   853  		}
   854  		return pconn, pconn.idleAt
   855  	}
   856  }
   857  
   858  // removeIdleConn marks pconn as dead.
   859  func (t *Transport) removeIdleConn(pconn *persistConn) {
   860  	t.idleMu.Lock()
   861  	defer t.idleMu.Unlock()
   862  	t.removeIdleConnLocked(pconn)
   863  }
   864  
   865  // t.idleMu must be held.
   866  func (t *Transport) removeIdleConnLocked(pconn *persistConn) {
   867  	if pconn.idleTimer != nil {
   868  		pconn.idleTimer.Stop()
   869  	}
   870  	t.idleLRU.remove(pconn)
   871  	key := pconn.cacheKey
   872  	pconns := t.idleConn[key]
   873  	switch len(pconns) {
   874  	case 0:
   875  		// Nothing
   876  	case 1:
   877  		if pconns[0] == pconn {
   878  			delete(t.idleConn, key)
   879  		}
   880  	default:
   881  		for i, v := range pconns {
   882  			if v != pconn {
   883  				continue
   884  			}
   885  			// Slide down, keeping most recently-used
   886  			// conns at the end.
   887  			copy(pconns[i:], pconns[i+1:])
   888  			t.idleConn[key] = pconns[:len(pconns)-1]
   889  			break
   890  		}
   891  	}
   892  }
   893  
   894  func (t *Transport) setReqCanceler(r *Request, fn func(error)) {
   895  	t.reqMu.Lock()
   896  	defer t.reqMu.Unlock()
   897  	if t.reqCanceler == nil {
   898  		t.reqCanceler = make(map[*Request]func(error))
   899  	}
   900  	if fn != nil {
   901  		t.reqCanceler[r] = fn
   902  	} else {
   903  		delete(t.reqCanceler, r)
   904  	}
   905  }
   906  
   907  // replaceReqCanceler replaces an existing cancel function. If there is no cancel function
   908  // for the request, we don't set the function and return false.
   909  // Since CancelRequest will clear the canceler, we can use the return value to detect if
   910  // the request was canceled since the last setReqCancel call.
   911  func (t *Transport) replaceReqCanceler(r *Request, fn func(error)) bool {
   912  	t.reqMu.Lock()
   913  	defer t.reqMu.Unlock()
   914  	_, ok := t.reqCanceler[r]
   915  	if !ok {
   916  		return false
   917  	}
   918  	if fn != nil {
   919  		t.reqCanceler[r] = fn
   920  	} else {
   921  		delete(t.reqCanceler, r)
   922  	}
   923  	return true
   924  }
   925  
   926  var zeroDialer net.Dialer
   927  
   928  func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) {
   929  	if t.DialContext != nil {
   930  		return t.DialContext(ctx, network, addr)
   931  	}
   932  	if t.Dial != nil {
   933  		c, err := t.Dial(network, addr)
   934  		if c == nil && err == nil {
   935  			err = errors.New("net/http: Transport.Dial hook returned (nil, nil)")
   936  		}
   937  		return c, err
   938  	}
   939  	return zeroDialer.DialContext(ctx, network, addr)
   940  }
   941  
   942  // getConn dials and creates a new persistConn to the target as
   943  // specified in the connectMethod. This includes doing a proxy CONNECT
   944  // and/or setting up TLS.  If this doesn't return an error, the persistConn
   945  // is ready to write requests to.
   946  func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) {
   947  	req := treq.Request
   948  	trace := treq.trace
   949  	ctx := req.Context()
   950  	if trace != nil && trace.GetConn != nil {
   951  		trace.GetConn(cm.addr())
   952  	}
   953  	if pc, idleSince := t.getIdleConn(cm); pc != nil {
   954  		if trace != nil && trace.GotConn != nil {
   955  			trace.GotConn(pc.gotIdleConnTrace(idleSince))
   956  		}
   957  		// set request canceler to some non-nil function so we
   958  		// can detect whether it was cleared between now and when
   959  		// we enter roundTrip
   960  		t.setReqCanceler(req, func(error) {})
   961  		return pc, nil
   962  	}
   963  
   964  	type dialRes struct {
   965  		pc  *persistConn
   966  		err error
   967  	}
   968  	dialc := make(chan dialRes)
   969  
   970  	// Copy these hooks so we don't race on the postPendingDial in
   971  	// the goroutine we launch. Issue 11136.
   972  	testHookPrePendingDial := testHookPrePendingDial
   973  	testHookPostPendingDial := testHookPostPendingDial
   974  
   975  	handlePendingDial := func() {
   976  		testHookPrePendingDial()
   977  		go func() {
   978  			if v := <-dialc; v.err == nil {
   979  				t.putOrCloseIdleConn(v.pc)
   980  			}
   981  			testHookPostPendingDial()
   982  		}()
   983  	}
   984  
   985  	cancelc := make(chan error, 1)
   986  	t.setReqCanceler(req, func(err error) { cancelc <- err })
   987  
   988  	go func() {
   989  		pc, err := t.dialConn(ctx, cm)
   990  		dialc <- dialRes{pc, err}
   991  	}()
   992  
   993  	idleConnCh := t.getIdleConnCh(cm)
   994  	select {
   995  	case v := <-dialc:
   996  		// Our dial finished.
   997  		if v.pc != nil {
   998  			if trace != nil && trace.GotConn != nil && v.pc.alt == nil {
   999  				trace.GotConn(httptrace.GotConnInfo{Conn: v.pc.conn})
  1000  			}
  1001  			return v.pc, nil
  1002  		}
  1003  		// Our dial failed. See why to return a nicer error
  1004  		// value.
  1005  		select {
  1006  		case <-req.Cancel:
  1007  			// It was an error due to cancelation, so prioritize that
  1008  			// error value. (Issue 16049)
  1009  			return nil, errRequestCanceledConn
  1010  		case <-req.Context().Done():
  1011  			return nil, req.Context().Err()
  1012  		case err := <-cancelc:
  1013  			if err == errRequestCanceled {
  1014  				err = errRequestCanceledConn
  1015  			}
  1016  			return nil, err
  1017  		default:
  1018  			// It wasn't an error due to cancelation, so
  1019  			// return the original error message:
  1020  			return nil, v.err
  1021  		}
  1022  	case pc := <-idleConnCh:
  1023  		// Another request finished first and its net.Conn
  1024  		// became available before our dial. Or somebody
  1025  		// else's dial that they didn't use.
  1026  		// But our dial is still going, so give it away
  1027  		// when it finishes:
  1028  		handlePendingDial()
  1029  		if trace != nil && trace.GotConn != nil {
  1030  			trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()})
  1031  		}
  1032  		return pc, nil
  1033  	case <-req.Cancel:
  1034  		handlePendingDial()
  1035  		return nil, errRequestCanceledConn
  1036  	case <-req.Context().Done():
  1037  		handlePendingDial()
  1038  		return nil, req.Context().Err()
  1039  	case err := <-cancelc:
  1040  		handlePendingDial()
  1041  		if err == errRequestCanceled {
  1042  			err = errRequestCanceledConn
  1043  		}
  1044  		return nil, err
  1045  	}
  1046  }
  1047  
  1048  type oneConnDialer <-chan net.Conn
  1049  
  1050  func newOneConnDialer(c net.Conn) proxy.Dialer {
  1051  	ch := make(chan net.Conn, 1)
  1052  	ch <- c
  1053  	return oneConnDialer(ch)
  1054  }
  1055  
  1056  func (d oneConnDialer) Dial(network, addr string) (net.Conn, error) {
  1057  	select {
  1058  	case c := <-d:
  1059  		return c, nil
  1060  	default:
  1061  		return nil, io.EOF
  1062  	}
  1063  }
  1064  
  1065  // The connect method and the transport can both specify a TLS
  1066  // Host name.  The transport's name takes precedence if present.
  1067  func chooseTLSHost(cm connectMethod, t *Transport) string {
  1068  	tlsHost := ""
  1069  	if t.TLSClientConfig != nil {
  1070  		tlsHost = t.TLSClientConfig.ServerName
  1071  	}
  1072  	if tlsHost == "" {
  1073  		tlsHost = cm.tlsHost()
  1074  	}
  1075  	return tlsHost
  1076  }
  1077  
  1078  // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS
  1079  // tunnel, this function establishes a nested TLS session inside the encrypted channel.
  1080  // The remote endpoint's name may be overridden by TLSClientConfig.ServerName.
  1081  func (pconn *persistConn) addTLS(name string, trace *httptrace.ClientTrace) error {
  1082  	// Initiate TLS and check remote host name against certificate.
  1083  	cfg := cloneTLSConfig(pconn.t.TLSClientConfig)
  1084  	if cfg.ServerName == "" {
  1085  		cfg.ServerName = name
  1086  	}
  1087  	plainConn := pconn.conn
  1088  	tlsConn := tls.Client(plainConn, cfg)
  1089  	errc := make(chan error, 2)
  1090  	var timer *time.Timer // for canceling TLS handshake
  1091  	if d := pconn.t.TLSHandshakeTimeout; d != 0 {
  1092  		timer = time.AfterFunc(d, func() {
  1093  			errc <- tlsHandshakeTimeoutError{}
  1094  		})
  1095  	}
  1096  	go func() {
  1097  		if trace != nil && trace.TLSHandshakeStart != nil {
  1098  			trace.TLSHandshakeStart()
  1099  		}
  1100  		err := tlsConn.Handshake()
  1101  		if timer != nil {
  1102  			timer.Stop()
  1103  		}
  1104  		errc <- err
  1105  	}()
  1106  	if err := <-errc; err != nil {
  1107  		plainConn.Close()
  1108  		if trace != nil && trace.TLSHandshakeDone != nil {
  1109  			trace.TLSHandshakeDone(tls.ConnectionState{}, err)
  1110  		}
  1111  		return err
  1112  	}
  1113  	if !cfg.InsecureSkipVerify {
  1114  		if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
  1115  			plainConn.Close()
  1116  			return err
  1117  		}
  1118  	}
  1119  	cs := tlsConn.ConnectionState()
  1120  	if trace != nil && trace.TLSHandshakeDone != nil {
  1121  		trace.TLSHandshakeDone(cs, nil)
  1122  	}
  1123  	pconn.tlsState = &cs
  1124  	pconn.conn = tlsConn
  1125  	return nil
  1126  }
  1127  
  1128  func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) {
  1129  	pconn := &persistConn{
  1130  		t:             t,
  1131  		cacheKey:      cm.key(),
  1132  		reqch:         make(chan requestAndChan, 1),
  1133  		writech:       make(chan writeRequest, 1),
  1134  		closech:       make(chan struct{}),
  1135  		writeErrCh:    make(chan error, 1),
  1136  		writeLoopDone: make(chan struct{}),
  1137  	}
  1138  	trace := httptrace.ContextClientTrace(ctx)
  1139  	wrapErr := func(err error) error {
  1140  		if cm.proxyURL != nil {
  1141  			// Return a typed error, per Issue 16997
  1142  			return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err}
  1143  		}
  1144  		return err
  1145  	}
  1146  	if cm.scheme() == "https" && t.DialTLS != nil {
  1147  		var err error
  1148  		pconn.conn, err = t.DialTLS("tcp", cm.addr())
  1149  		if err != nil {
  1150  			return nil, wrapErr(err)
  1151  		}
  1152  		if pconn.conn == nil {
  1153  			return nil, wrapErr(errors.New("net/http: Transport.DialTLS returned (nil, nil)"))
  1154  		}
  1155  		if tc, ok := pconn.conn.(*tls.Conn); ok {
  1156  			// Handshake here, in case DialTLS didn't. TLSNextProto below
  1157  			// depends on it for knowing the connection state.
  1158  			if trace != nil && trace.TLSHandshakeStart != nil {
  1159  				trace.TLSHandshakeStart()
  1160  			}
  1161  			if err := tc.Handshake(); err != nil {
  1162  				go pconn.conn.Close()
  1163  				if trace != nil && trace.TLSHandshakeDone != nil {
  1164  					trace.TLSHandshakeDone(tls.ConnectionState{}, err)
  1165  				}
  1166  				return nil, err
  1167  			}
  1168  			cs := tc.ConnectionState()
  1169  			if trace != nil && trace.TLSHandshakeDone != nil {
  1170  				trace.TLSHandshakeDone(cs, nil)
  1171  			}
  1172  			pconn.tlsState = &cs
  1173  		}
  1174  	} else {
  1175  		conn, err := t.dial(ctx, "tcp", cm.addr())
  1176  		if err != nil {
  1177  			return nil, wrapErr(err)
  1178  		}
  1179  		pconn.conn = conn
  1180  		if cm.scheme() == "https" {
  1181  			var firstTLSHost string
  1182  			if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil {
  1183  				return nil, wrapErr(err)
  1184  			}
  1185  			if err = pconn.addTLS(firstTLSHost, trace); err != nil {
  1186  				return nil, wrapErr(err)
  1187  			}
  1188  		}
  1189  	}
  1190  
  1191  	// Proxy setup.
  1192  	switch {
  1193  	case cm.proxyURL == nil:
  1194  		// Do nothing. Not using a proxy.
  1195  	case cm.proxyURL.Scheme == "socks5":
  1196  		conn := pconn.conn
  1197  		var auth *proxy.Auth
  1198  		if u := cm.proxyURL.User; u != nil {
  1199  			auth = &proxy.Auth{}
  1200  			auth.User = u.Username()
  1201  			auth.Password, _ = u.Password()
  1202  		}
  1203  		p, err := proxy.SOCKS5("", cm.addr(), auth, newOneConnDialer(conn))
  1204  		if err != nil {
  1205  			conn.Close()
  1206  			return nil, err
  1207  		}
  1208  		if _, err := p.Dial("tcp", cm.targetAddr); err != nil {
  1209  			conn.Close()
  1210  			return nil, err
  1211  		}
  1212  	case cm.targetScheme == "http":
  1213  		pconn.isProxy = true
  1214  		if pa := cm.proxyAuth(); pa != "" {
  1215  			pconn.mutateHeaderFunc = func(h Header) {
  1216  				h.Set("Proxy-Authorization", pa)
  1217  			}
  1218  		}
  1219  	case cm.targetScheme == "https":
  1220  		conn := pconn.conn
  1221  		hdr := t.ProxyConnectHeader
  1222  		if hdr == nil {
  1223  			hdr = make(Header)
  1224  		}
  1225  		connectReq := &Request{
  1226  			Method: "CONNECT",
  1227  			URL:    &url.URL{Opaque: cm.targetAddr},
  1228  			Host:   cm.targetAddr,
  1229  			Header: hdr,
  1230  		}
  1231  		if pa := cm.proxyAuth(); pa != "" {
  1232  			connectReq.Header.Set("Proxy-Authorization", pa)
  1233  		}
  1234  		connectReq.Write(conn)
  1235  
  1236  		// Read response.
  1237  		// Okay to use and discard buffered reader here, because
  1238  		// TLS server will not speak until spoken to.
  1239  		br := bufio.NewReader(conn)
  1240  		resp, err := ReadResponse(br, connectReq)
  1241  		if err != nil {
  1242  			conn.Close()
  1243  			return nil, err
  1244  		}
  1245  		if resp.StatusCode != 200 {
  1246  			f := strings.SplitN(resp.Status, " ", 2)
  1247  			conn.Close()
  1248  			if len(f) < 2 {
  1249  				return nil, errors.New("unknown status code")
  1250  			}
  1251  			return nil, errors.New(f[1])
  1252  		}
  1253  	}
  1254  
  1255  	if cm.proxyURL != nil && cm.targetScheme == "https" {
  1256  		if err := pconn.addTLS(cm.tlsHost(), trace); err != nil {
  1257  			return nil, err
  1258  		}
  1259  	}
  1260  
  1261  	if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
  1262  		if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
  1263  			return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil
  1264  		}
  1265  	}
  1266  
  1267  	pconn.br = bufio.NewReader(pconn)
  1268  	pconn.bw = bufio.NewWriter(persistConnWriter{pconn})
  1269  	go pconn.readLoop()
  1270  	go pconn.writeLoop()
  1271  	return pconn, nil
  1272  }
  1273  
  1274  // persistConnWriter is the io.Writer written to by pc.bw.
  1275  // It accumulates the number of bytes written to the underlying conn,
  1276  // so the retry logic can determine whether any bytes made it across
  1277  // the wire.
  1278  // This is exactly 1 pointer field wide so it can go into an interface
  1279  // without allocation.
  1280  type persistConnWriter struct {
  1281  	pc *persistConn
  1282  }
  1283  
  1284  func (w persistConnWriter) Write(p []byte) (n int, err error) {
  1285  	n, err = w.pc.conn.Write(p)
  1286  	w.pc.nwrite += int64(n)
  1287  	return
  1288  }
  1289  
  1290  // useProxy reports whether requests to addr should use a proxy,
  1291  // according to the NO_PROXY or no_proxy environment variable.
  1292  // addr is always a canonicalAddr with a host and port.
  1293  func useProxy(addr string) bool {
  1294  	if len(addr) == 0 {
  1295  		return true
  1296  	}
  1297  	host, _, err := net.SplitHostPort(addr)
  1298  	if err != nil {
  1299  		return false
  1300  	}
  1301  	if host == "localhost" {
  1302  		return false
  1303  	}
  1304  	if ip := net.ParseIP(host); ip != nil {
  1305  		if ip.IsLoopback() {
  1306  			return false
  1307  		}
  1308  	}
  1309  
  1310  	noProxy := noProxyEnv.Get()
  1311  	if noProxy == "*" {
  1312  		return false
  1313  	}
  1314  
  1315  	addr = strings.ToLower(strings.TrimSpace(addr))
  1316  	if hasPort(addr) {
  1317  		addr = addr[:strings.LastIndex(addr, ":")]
  1318  	}
  1319  
  1320  	for _, p := range strings.Split(noProxy, ",") {
  1321  		p = strings.ToLower(strings.TrimSpace(p))
  1322  		if len(p) == 0 {
  1323  			continue
  1324  		}
  1325  		if hasPort(p) {
  1326  			p = p[:strings.LastIndex(p, ":")]
  1327  		}
  1328  		if addr == p {
  1329  			return false
  1330  		}
  1331  		if len(p) == 0 {
  1332  			// There is no host part, likely the entry is malformed; ignore.
  1333  			continue
  1334  		}
  1335  		if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
  1336  			// no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
  1337  			return false
  1338  		}
  1339  		if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
  1340  			// no_proxy "foo.com" matches "bar.foo.com"
  1341  			return false
  1342  		}
  1343  	}
  1344  	return true
  1345  }
  1346  
  1347  // connectMethod is the map key (in its String form) for keeping persistent
  1348  // TCP connections alive for subsequent HTTP requests.
  1349  //
  1350  // A connect method may be of the following types:
  1351  //
  1352  //	Cache key form                    Description
  1353  //	-----------------                 -------------------------
  1354  //	|http|foo.com                     http directly to server, no proxy
  1355  //	|https|foo.com                    https directly to server, no proxy
  1356  //	http://proxy.com|https|foo.com    http to proxy, then CONNECT to foo.com
  1357  //	http://proxy.com|http             http to proxy, http to anywhere after that
  1358  //	socks5://proxy.com|http|foo.com   socks5 to proxy, then http to foo.com
  1359  //	socks5://proxy.com|https|foo.com  socks5 to proxy, then https to foo.com
  1360  //	https://proxy.com|https|foo.com   https to proxy, then CONNECT to foo.com
  1361  //	https://proxy.com|http            https to proxy, http to anywhere after that
  1362  //
  1363  type connectMethod struct {
  1364  	proxyURL     *url.URL // nil for no proxy, else full proxy URL
  1365  	targetScheme string   // "http" or "https"
  1366  	// If proxyURL specifies an http or https proxy, and targetScheme is http (not https),
  1367  	// then targetAddr is not included in the connect method key, because the socket can
  1368  	// be reused for different targetAddr values.
  1369  	targetAddr string
  1370  
  1371  	GetConnectMethodAddr func(addr string) string
  1372  }
  1373  
  1374  func (cm *connectMethod) key() connectMethodKey {
  1375  	proxyStr := ""
  1376  	targetAddr := cm.targetAddr
  1377  	if cm.GetConnectMethodAddr != nil {
  1378  		targetAddr = cm.GetConnectMethodAddr(targetAddr)
  1379  	}
  1380  	if cm.proxyURL != nil {
  1381  		proxyStr = cm.proxyURL.String()
  1382  		if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" {
  1383  			targetAddr = ""
  1384  		}
  1385  	}
  1386  	return connectMethodKey{
  1387  		proxy:  proxyStr,
  1388  		scheme: cm.targetScheme,
  1389  		addr:   targetAddr,
  1390  	}
  1391  }
  1392  
  1393  // scheme returns the first hop scheme: http, https, or socks5
  1394  func (cm *connectMethod) scheme() string {
  1395  	if cm.proxyURL != nil {
  1396  		return cm.proxyURL.Scheme
  1397  	}
  1398  	return cm.targetScheme
  1399  }
  1400  
  1401  // addr returns the first hop "host:port" to which we need to TCP connect.
  1402  func (cm *connectMethod) addr() string {
  1403  	if cm.proxyURL != nil {
  1404  		return canonicalAddr(cm.proxyURL)
  1405  	}
  1406  	return cm.targetAddr
  1407  }
  1408  
  1409  // tlsHost returns the host name to match against the peer's
  1410  // TLS certificate.
  1411  func (cm *connectMethod) tlsHost() string {
  1412  	h := cm.targetAddr
  1413  	if hasPort(h) {
  1414  		h = h[:strings.LastIndex(h, ":")]
  1415  	}
  1416  	return h
  1417  }
  1418  
  1419  // connectMethodKey is the map key version of connectMethod, with a
  1420  // stringified proxy URL (or the empty string) instead of a pointer to
  1421  // a URL.
  1422  type connectMethodKey struct {
  1423  	proxy, scheme, addr string
  1424  }
  1425  
  1426  func (k connectMethodKey) String() string {
  1427  	// Only used by tests.
  1428  	return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr)
  1429  }
  1430  
  1431  // persistConn wraps a connection, usually a persistent one
  1432  // (but may be used for non-keep-alive requests as well)
  1433  type persistConn struct {
  1434  	// alt optionally specifies the TLS NextProto RoundTripper.
  1435  	// This is used for HTTP/2 today and future protocols later.
  1436  	// If it's non-nil, the rest of the fields are unused.
  1437  	alt RoundTripper
  1438  
  1439  	t         *Transport
  1440  	cacheKey  connectMethodKey
  1441  	conn      net.Conn
  1442  	tlsState  *tls.ConnectionState
  1443  	br        *bufio.Reader       // from conn
  1444  	bw        *bufio.Writer       // to conn
  1445  	nwrite    int64               // bytes written
  1446  	reqch     chan requestAndChan // written by roundTrip; read by readLoop
  1447  	writech   chan writeRequest   // written by roundTrip; read by writeLoop
  1448  	closech   chan struct{}       // closed when conn closed
  1449  	isProxy   bool
  1450  	sawEOF    bool  // whether we've seen EOF from conn; owned by readLoop
  1451  	readLimit int64 // bytes allowed to be read; owned by readLoop
  1452  	// writeErrCh passes the request write error (usually nil)
  1453  	// from the writeLoop goroutine to the readLoop which passes
  1454  	// it off to the res.Body reader, which then uses it to decide
  1455  	// whether or not a connection can be reused. Issue 7569.
  1456  	writeErrCh chan error
  1457  
  1458  	writeLoopDone chan struct{} // closed when write loop ends
  1459  
  1460  	// Both guarded by Transport.idleMu:
  1461  	idleAt    time.Time   // time it last become idle
  1462  	idleTimer *time.Timer // holding an AfterFunc to close it
  1463  
  1464  	mu                   sync.Mutex // guards following fields
  1465  	numExpectedResponses int
  1466  	closed               error // set non-nil when conn is closed, before closech is closed
  1467  	canceledErr          error // set non-nil if conn is canceled
  1468  	broken               bool  // an error has happened on this connection; marked broken so it's not reused.
  1469  	reused               bool  // whether conn has had successful request/response and is being reused.
  1470  	// mutateHeaderFunc is an optional func to modify extra
  1471  	// headers on each outbound request before it's written. (the
  1472  	// original Request given to RoundTrip is not modified)
  1473  	mutateHeaderFunc func(Header)
  1474  }
  1475  
  1476  func (pc *persistConn) maxHeaderResponseSize() int64 {
  1477  	if v := pc.t.MaxResponseHeaderBytes; v != 0 {
  1478  		return v
  1479  	}
  1480  	return 10 << 20 // conservative default; same as http2
  1481  }
  1482  
  1483  func (pc *persistConn) Read(p []byte) (n int, err error) {
  1484  	if pc.readLimit <= 0 {
  1485  		return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize())
  1486  	}
  1487  	if int64(len(p)) > pc.readLimit {
  1488  		p = p[:pc.readLimit]
  1489  	}
  1490  	n, err = pc.conn.Read(p)
  1491  	if err == io.EOF {
  1492  		pc.sawEOF = true
  1493  	}
  1494  	pc.readLimit -= int64(n)
  1495  	return
  1496  }
  1497  
  1498  // isBroken reports whether this connection is in a known broken state.
  1499  func (pc *persistConn) isBroken() bool {
  1500  	pc.mu.Lock()
  1501  	b := pc.closed != nil
  1502  	pc.mu.Unlock()
  1503  	return b
  1504  }
  1505  
  1506  // canceled returns non-nil if the connection was closed due to
  1507  // CancelRequest or due to context cancelation.
  1508  func (pc *persistConn) canceled() error {
  1509  	pc.mu.Lock()
  1510  	defer pc.mu.Unlock()
  1511  	return pc.canceledErr
  1512  }
  1513  
  1514  // isReused reports whether this connection is in a known broken state.
  1515  func (pc *persistConn) isReused() bool {
  1516  	pc.mu.Lock()
  1517  	r := pc.reused
  1518  	pc.mu.Unlock()
  1519  	return r
  1520  }
  1521  
  1522  func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) {
  1523  	pc.mu.Lock()
  1524  	defer pc.mu.Unlock()
  1525  	t.Reused = pc.reused
  1526  	t.Conn = pc.conn
  1527  	t.WasIdle = true
  1528  	if !idleAt.IsZero() {
  1529  		t.IdleTime = time.Since(idleAt)
  1530  	}
  1531  	return
  1532  }
  1533  
  1534  func (pc *persistConn) cancelRequest(err error) {
  1535  	pc.mu.Lock()
  1536  	defer pc.mu.Unlock()
  1537  	pc.canceledErr = err
  1538  	pc.closeLocked(errRequestCanceled)
  1539  }
  1540  
  1541  // closeConnIfStillIdle closes the connection if it's still sitting idle.
  1542  // This is what's called by the persistConn's idleTimer, and is run in its
  1543  // own goroutine.
  1544  func (pc *persistConn) closeConnIfStillIdle() {
  1545  	t := pc.t
  1546  	t.idleMu.Lock()
  1547  	defer t.idleMu.Unlock()
  1548  	if _, ok := t.idleLRU.m[pc]; !ok {
  1549  		// Not idle.
  1550  		return
  1551  	}
  1552  	t.removeIdleConnLocked(pc)
  1553  	pc.close(errIdleConnTimeout)
  1554  }
  1555  
  1556  // mapRoundTripError returns the appropriate error value for
  1557  // persistConn.roundTrip.
  1558  //
  1559  // The provided err is the first error that (*persistConn).roundTrip
  1560  // happened to receive from its select statement.
  1561  //
  1562  // The startBytesWritten value should be the value of pc.nwrite before the roundTrip
  1563  // started writing the request.
  1564  func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error {
  1565  	if err == nil {
  1566  		return nil
  1567  	}
  1568  
  1569  	// If the request was canceled, that's better than network
  1570  	// failures that were likely the result of tearing down the
  1571  	// connection.
  1572  	if cerr := pc.canceled(); cerr != nil {
  1573  		return cerr
  1574  	}
  1575  
  1576  	// See if an error was set explicitly.
  1577  	req.mu.Lock()
  1578  	reqErr := req.err
  1579  	req.mu.Unlock()
  1580  	if reqErr != nil {
  1581  		return reqErr
  1582  	}
  1583  
  1584  	if err == errServerClosedIdle {
  1585  		// Don't decorate
  1586  		return err
  1587  	}
  1588  
  1589  	if _, ok := err.(transportReadFromServerError); ok {
  1590  		// Don't decorate
  1591  		return err
  1592  	}
  1593  	if pc.isBroken() {
  1594  		<-pc.writeLoopDone
  1595  		if pc.nwrite == startBytesWritten {
  1596  			return nothingWrittenError{err}
  1597  		}
  1598  		return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err)
  1599  	}
  1600  	return err
  1601  }
  1602  
  1603  func (pc *persistConn) readLoop() {
  1604  	closeErr := errReadLoopExiting // default value, if not changed below
  1605  	defer func() {
  1606  		pc.close(closeErr)
  1607  		pc.t.removeIdleConn(pc)
  1608  	}()
  1609  
  1610  	tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {
  1611  		if err := pc.t.tryPutIdleConn(pc); err != nil {
  1612  			closeErr = err
  1613  			if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
  1614  				trace.PutIdleConn(err)
  1615  			}
  1616  			return false
  1617  		}
  1618  		if trace != nil && trace.PutIdleConn != nil {
  1619  			trace.PutIdleConn(nil)
  1620  		}
  1621  		return true
  1622  	}
  1623  
  1624  	// eofc is used to block caller goroutines reading from Response.Body
  1625  	// at EOF until this goroutines has (potentially) added the connection
  1626  	// back to the idle pool.
  1627  	eofc := make(chan struct{})
  1628  	defer close(eofc) // unblock reader on errors
  1629  
  1630  	// Read this once, before loop starts. (to avoid races in tests)
  1631  	testHookMu.Lock()
  1632  	testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
  1633  	testHookMu.Unlock()
  1634  
  1635  	alive := true
  1636  	for alive {
  1637  		pc.readLimit = pc.maxHeaderResponseSize()
  1638  		_, err := pc.br.Peek(1)
  1639  
  1640  		pc.mu.Lock()
  1641  		if pc.numExpectedResponses == 0 {
  1642  			pc.readLoopPeekFailLocked(err)
  1643  			pc.mu.Unlock()
  1644  			return
  1645  		}
  1646  		pc.mu.Unlock()
  1647  
  1648  		rc := <-pc.reqch
  1649  		trace := httptrace.ContextClientTrace(rc.req.Context())
  1650  
  1651  		var resp *Response
  1652  		if err == nil {
  1653  			resp, err = pc.readResponse(rc, trace)
  1654  		} else {
  1655  			err = transportReadFromServerError{err}
  1656  			closeErr = err
  1657  		}
  1658  
  1659  		if err != nil {
  1660  			if pc.readLimit <= 0 {
  1661  				err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
  1662  			}
  1663  
  1664  			select {
  1665  			case rc.ch <- responseAndError{err: err}:
  1666  			case <-rc.callerGone:
  1667  				return
  1668  			}
  1669  			return
  1670  		}
  1671  		pc.readLimit = maxInt64 // effictively no limit for response bodies
  1672  
  1673  		pc.mu.Lock()
  1674  		pc.numExpectedResponses--
  1675  		pc.mu.Unlock()
  1676  
  1677  		hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
  1678  
  1679  		if resp.Close || rc.req.Close || resp.StatusCode <= 199 {
  1680  			// Don't do keep-alive on error if either party requested a close
  1681  			// or we get an unexpected informational (1xx) response.
  1682  			// StatusCode 100 is already handled above.
  1683  			alive = false
  1684  		}
  1685  
  1686  		if !hasBody {
  1687  			pc.t.setReqCanceler(rc.req, nil)
  1688  
  1689  			// Put the idle conn back into the pool before we send the response
  1690  			// so if they process it quickly and make another request, they'll
  1691  			// get this same conn. But we use the unbuffered channel 'rc'
  1692  			// to guarantee that persistConn.roundTrip got out of its select
  1693  			// potentially waiting for this persistConn to close.
  1694  			// but after
  1695  			alive = alive &&
  1696  				!pc.sawEOF &&
  1697  				pc.wroteRequest() &&
  1698  				tryPutIdleConn(trace)
  1699  
  1700  			select {
  1701  			case rc.ch <- responseAndError{res: resp}:
  1702  			case <-rc.callerGone:
  1703  				return
  1704  			}
  1705  
  1706  			// Now that they've read from the unbuffered channel, they're safely
  1707  			// out of the select that also waits on this goroutine to die, so
  1708  			// we're allowed to exit now if needed (if alive is false)
  1709  			testHookReadLoopBeforeNextRead()
  1710  			continue
  1711  		}
  1712  
  1713  		waitForBodyRead := make(chan bool, 2)
  1714  		body := &bodyEOFSignal{
  1715  			body: resp.Body,
  1716  			earlyCloseFn: func() error {
  1717  				waitForBodyRead <- false
  1718  				<-eofc // will be closed by deferred call at the end of the function
  1719  				return nil
  1720  
  1721  			},
  1722  			fn: func(err error) error {
  1723  				isEOF := err == io.EOF
  1724  				waitForBodyRead <- isEOF
  1725  				if isEOF {
  1726  					<-eofc // see comment above eofc declaration
  1727  				} else if err != nil {
  1728  					if cerr := pc.canceled(); cerr != nil {
  1729  						return cerr
  1730  					}
  1731  				}
  1732  				return err
  1733  			},
  1734  		}
  1735  
  1736  		resp.Body = body
  1737  		if rc.addedGzip && strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
  1738  			resp.Body = &gzipReader{body: body}
  1739  			resp.Header.Del("Content-Encoding")
  1740  			resp.Header.Del("Content-Length")
  1741  			resp.ContentLength = -1
  1742  			resp.Uncompressed = true
  1743  		}
  1744  
  1745  		select {
  1746  		case rc.ch <- responseAndError{res: resp}:
  1747  		case <-rc.callerGone:
  1748  			return
  1749  		}
  1750  
  1751  		// Before looping back to the top of this function and peeking on
  1752  		// the bufio.Reader, wait for the caller goroutine to finish
  1753  		// reading the response body. (or for cancelation or death)
  1754  		select {
  1755  		case bodyEOF := <-waitForBodyRead:
  1756  			pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool
  1757  			alive = alive &&
  1758  				bodyEOF &&
  1759  				!pc.sawEOF &&
  1760  				pc.wroteRequest() &&
  1761  				tryPutIdleConn(trace)
  1762  			if bodyEOF {
  1763  				eofc <- struct{}{}
  1764  			}
  1765  		case <-rc.req.Cancel:
  1766  			alive = false
  1767  			pc.t.CancelRequest(rc.req)
  1768  		case <-rc.req.Context().Done():
  1769  			alive = false
  1770  			pc.t.cancelRequest(rc.req, rc.req.Context().Err())
  1771  		case <-pc.closech:
  1772  			alive = false
  1773  		}
  1774  
  1775  		testHookReadLoopBeforeNextRead()
  1776  	}
  1777  }
  1778  
  1779  func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
  1780  	if pc.closed != nil {
  1781  		return
  1782  	}
  1783  	if n := pc.br.Buffered(); n > 0 {
  1784  		buf, _ := pc.br.Peek(n)
  1785  		log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
  1786  	}
  1787  	if peekErr == io.EOF {
  1788  		// common case.
  1789  		pc.closeLocked(errServerClosedIdle)
  1790  	} else {
  1791  		pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr))
  1792  	}
  1793  }
  1794  
  1795  // readResponse reads an HTTP response (or two, in the case of "Expect:
  1796  // 100-continue") from the server. It returns the final non-100 one.
  1797  // trace is optional.
  1798  func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) {
  1799  	if trace != nil && trace.GotFirstResponseByte != nil {
  1800  		if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 {
  1801  			trace.GotFirstResponseByte()
  1802  		}
  1803  	}
  1804  	resp, err = ReadResponse(pc.br, rc.req)
  1805  	if err != nil {
  1806  		return
  1807  	}
  1808  	if rc.continueCh != nil {
  1809  		if resp.StatusCode == 100 {
  1810  			if trace != nil && trace.Got100Continue != nil {
  1811  				trace.Got100Continue()
  1812  			}
  1813  			rc.continueCh <- struct{}{}
  1814  		} else {
  1815  			close(rc.continueCh)
  1816  		}
  1817  	}
  1818  	if resp.StatusCode == 100 {
  1819  		pc.readLimit = pc.maxHeaderResponseSize() // reset the limit
  1820  		resp, err = ReadResponse(pc.br, rc.req)
  1821  		if err != nil {
  1822  			return
  1823  		}
  1824  	}
  1825  	resp.TLS = pc.tlsState
  1826  	return
  1827  }
  1828  
  1829  // waitForContinue returns the function to block until
  1830  // any response, timeout or connection close. After any of them,
  1831  // the function returns a bool which indicates if the body should be sent.
  1832  func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
  1833  	if continueCh == nil {
  1834  		return nil
  1835  	}
  1836  	return func() bool {
  1837  		timer := time.NewTimer(pc.t.ExpectContinueTimeout)
  1838  		defer timer.Stop()
  1839  
  1840  		select {
  1841  		case _, ok := <-continueCh:
  1842  			return ok
  1843  		case <-timer.C:
  1844  			return true
  1845  		case <-pc.closech:
  1846  			return false
  1847  		}
  1848  	}
  1849  }
  1850  
  1851  // nothingWrittenError wraps a write errors which ended up writing zero bytes.
  1852  type nothingWrittenError struct {
  1853  	error
  1854  }
  1855  
  1856  func (pc *persistConn) writeLoop() {
  1857  	defer close(pc.writeLoopDone)
  1858  	for {
  1859  		select {
  1860  		case wr := <-pc.writech:
  1861  			startBytesWritten := pc.nwrite
  1862  			err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
  1863  			if bre, ok := err.(requestBodyReadError); ok {
  1864  				err = bre.error
  1865  				// Errors reading from the user's
  1866  				// Request.Body are high priority.
  1867  				// Set it here before sending on the
  1868  				// channels below or calling
  1869  				// pc.close() which tears town
  1870  				// connections and causes other
  1871  				// errors.
  1872  				wr.req.setError(err)
  1873  			}
  1874  			if err == nil {
  1875  				err = pc.bw.Flush()
  1876  			}
  1877  			if err != nil {
  1878  				wr.req.Request.closeBody()
  1879  				if pc.nwrite == startBytesWritten {
  1880  					err = nothingWrittenError{err}
  1881  				}
  1882  			}
  1883  			pc.writeErrCh <- err // to the body reader, which might recycle us
  1884  			wr.ch <- err         // to the roundTrip function
  1885  			if err != nil {
  1886  				pc.close(err)
  1887  				return
  1888  			}
  1889  		case <-pc.closech:
  1890  			return
  1891  		}
  1892  	}
  1893  }
  1894  
  1895  // wroteRequest is a check before recycling a connection that the previous write
  1896  // (from writeLoop above) happened and was successful.
  1897  func (pc *persistConn) wroteRequest() bool {
  1898  	select {
  1899  	case err := <-pc.writeErrCh:
  1900  		// Common case: the write happened well before the response, so
  1901  		// avoid creating a timer.
  1902  		return err == nil
  1903  	default:
  1904  		// Rare case: the request was written in writeLoop above but
  1905  		// before it could send to pc.writeErrCh, the reader read it
  1906  		// all, processed it, and called us here. In this case, give the
  1907  		// write goroutine a bit of time to finish its send.
  1908  		//
  1909  		// Less rare case: We also get here in the legitimate case of
  1910  		// Issue 7569, where the writer is still writing (or stalled),
  1911  		// but the server has already replied. In this case, we don't
  1912  		// want to wait too long, and we want to return false so this
  1913  		// connection isn't re-used.
  1914  		select {
  1915  		case err := <-pc.writeErrCh:
  1916  			return err == nil
  1917  		case <-time.After(50 * time.Millisecond):
  1918  			return false
  1919  		}
  1920  	}
  1921  }
  1922  
  1923  // responseAndError is how the goroutine reading from an HTTP/1 server
  1924  // communicates with the goroutine doing the RoundTrip.
  1925  type responseAndError struct {
  1926  	res *Response // else use this response (see res method)
  1927  	err error
  1928  }
  1929  
  1930  type requestAndChan struct {
  1931  	req *Request
  1932  	ch  chan responseAndError // unbuffered; always send in select on callerGone
  1933  
  1934  	// whether the Transport (as opposed to the user client code)
  1935  	// added the Accept-Encoding gzip header. If the Transport
  1936  	// set it, only then do we transparently decode the gzip.
  1937  	addedGzip bool
  1938  
  1939  	// Optional blocking chan for Expect: 100-continue (for send).
  1940  	// If the request has an "Expect: 100-continue" header and
  1941  	// the server responds 100 Continue, readLoop send a value
  1942  	// to writeLoop via this chan.
  1943  	continueCh chan<- struct{}
  1944  
  1945  	callerGone <-chan struct{} // closed when roundTrip caller has returned
  1946  }
  1947  
  1948  // A writeRequest is sent by the readLoop's goroutine to the
  1949  // writeLoop's goroutine to write a request while the read loop
  1950  // concurrently waits on both the write response and the server's
  1951  // reply.
  1952  type writeRequest struct {
  1953  	req *transportRequest
  1954  	ch  chan<- error
  1955  
  1956  	// Optional blocking chan for Expect: 100-continue (for receive).
  1957  	// If not nil, writeLoop blocks sending request body until
  1958  	// it receives from this chan.
  1959  	continueCh <-chan struct{}
  1960  }
  1961  
  1962  type httpError struct {
  1963  	err     string
  1964  	timeout bool
  1965  }
  1966  
  1967  func (e *httpError) Error() string   { return e.err }
  1968  func (e *httpError) Timeout() bool   { return e.timeout }
  1969  func (e *httpError) Temporary() bool { return true }
  1970  
  1971  var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
  1972  var errRequestCanceled = errors.New("net/http: request canceled")
  1973  var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify?
  1974  
  1975  func nop() {}
  1976  
  1977  // testHooks. Always non-nil.
  1978  var (
  1979  	testHookEnterRoundTrip   = nop
  1980  	testHookWaitResLoop      = nop
  1981  	testHookRoundTripRetried = nop
  1982  	testHookPrePendingDial   = nop
  1983  	testHookPostPendingDial  = nop
  1984  
  1985  	testHookMu                     sync.Locker = fakeLocker{} // guards following
  1986  	testHookReadLoopBeforeNextRead             = nop
  1987  )
  1988  
  1989  func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
  1990  	testHookEnterRoundTrip()
  1991  	if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) {
  1992  		pc.t.putOrCloseIdleConn(pc)
  1993  		return nil, errRequestCanceled
  1994  	}
  1995  	pc.mu.Lock()
  1996  	pc.numExpectedResponses++
  1997  	headerFn := pc.mutateHeaderFunc
  1998  	pc.mu.Unlock()
  1999  
  2000  	if headerFn != nil {
  2001  		headerFn(req.extraHeaders())
  2002  	}
  2003  
  2004  	// Ask for a compressed version if the caller didn't set their
  2005  	// own value for Accept-Encoding. We only attempt to
  2006  	// uncompress the gzip stream if we were the layer that
  2007  	// requested it.
  2008  	requestedGzip := false
  2009  	if !pc.t.DisableCompression &&
  2010  		req.Header.Get("Accept-Encoding") == "" &&
  2011  		req.Header.Get("Range") == "" &&
  2012  		req.Method != "HEAD" {
  2013  		// Request gzip only, not deflate. Deflate is ambiguous and
  2014  		// not as universally supported anyway.
  2015  		// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
  2016  		//
  2017  		// Note that we don't request this for HEAD requests,
  2018  		// due to a bug in nginx:
  2019  		//   http://trac.nginx.org/nginx/ticket/358
  2020  		//   https://golang.org/issue/5522
  2021  		//
  2022  		// We don't request gzip if the request is for a range, since
  2023  		// auto-decoding a portion of a gzipped document will just fail
  2024  		// anyway. See https://golang.org/issue/8923
  2025  		requestedGzip = true
  2026  		req.extraHeaders().Set("Accept-Encoding", "gzip")
  2027  	}
  2028  
  2029  	var continueCh chan struct{}
  2030  	if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
  2031  		continueCh = make(chan struct{}, 1)
  2032  	}
  2033  
  2034  	if pc.t.DisableKeepAlives {
  2035  		req.extraHeaders().Set("Connection", "close")
  2036  	}
  2037  
  2038  	gone := make(chan struct{})
  2039  	defer close(gone)
  2040  
  2041  	defer func() {
  2042  		if err != nil {
  2043  			pc.t.setReqCanceler(req.Request, nil)
  2044  		}
  2045  	}()
  2046  
  2047  	const debugRoundTrip = false
  2048  
  2049  	// Write the request concurrently with waiting for a response,
  2050  	// in case the server decides to reply before reading our full
  2051  	// request body.
  2052  	startBytesWritten := pc.nwrite
  2053  	writeErrCh := make(chan error, 1)
  2054  	pc.writech <- writeRequest{req, writeErrCh, continueCh}
  2055  
  2056  	resc := make(chan responseAndError)
  2057  	pc.reqch <- requestAndChan{
  2058  		req:        req.Request,
  2059  		ch:         resc,
  2060  		addedGzip:  requestedGzip,
  2061  		continueCh: continueCh,
  2062  		callerGone: gone,
  2063  	}
  2064  
  2065  	var respHeaderTimer <-chan time.Time
  2066  	cancelChan := req.Request.Cancel
  2067  	ctxDoneChan := req.Context().Done()
  2068  	for {
  2069  		testHookWaitResLoop()
  2070  		select {
  2071  		case err := <-writeErrCh:
  2072  			if debugRoundTrip {
  2073  				req.logf("writeErrCh resv: %T/%#v", err, err)
  2074  			}
  2075  			if err != nil {
  2076  				pc.close(fmt.Errorf("write error: %v", err))
  2077  				return nil, pc.mapRoundTripError(req, startBytesWritten, err)
  2078  			}
  2079  			if d := pc.t.ResponseHeaderTimeout; d > 0 {
  2080  				if debugRoundTrip {
  2081  					req.logf("starting timer for %v", d)
  2082  				}
  2083  				timer := time.NewTimer(d)
  2084  				defer timer.Stop() // prevent leaks
  2085  				respHeaderTimer = timer.C
  2086  			}
  2087  		case <-pc.closech:
  2088  			if debugRoundTrip {
  2089  				req.logf("closech recv: %T %#v", pc.closed, pc.closed)
  2090  			}
  2091  			return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed)
  2092  		case <-respHeaderTimer:
  2093  			if debugRoundTrip {
  2094  				req.logf("timeout waiting for response headers.")
  2095  			}
  2096  			pc.close(errTimeout)
  2097  			return nil, errTimeout
  2098  		case re := <-resc:
  2099  			if (re.res == nil) == (re.err == nil) {
  2100  				panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil))
  2101  			}
  2102  			if debugRoundTrip {
  2103  				req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err)
  2104  			}
  2105  			if re.err != nil {
  2106  				return nil, pc.mapRoundTripError(req, startBytesWritten, re.err)
  2107  			}
  2108  			return re.res, nil
  2109  		case <-cancelChan:
  2110  			pc.t.CancelRequest(req.Request)
  2111  			cancelChan = nil
  2112  		case <-ctxDoneChan:
  2113  			pc.t.cancelRequest(req.Request, req.Context().Err())
  2114  			cancelChan = nil
  2115  			ctxDoneChan = nil
  2116  		}
  2117  	}
  2118  }
  2119  
  2120  // tLogKey is a context WithValue key for test debugging contexts containing
  2121  // a t.Logf func. See export_test.go's Request.WithT method.
  2122  type tLogKey struct{}
  2123  
  2124  func (tr *transportRequest) logf(format string, args ...interface{}) {
  2125  	if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...interface{})); ok {
  2126  		logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...)
  2127  	}
  2128  }
  2129  
  2130  // markReused marks this connection as having been successfully used for a
  2131  // request and response.
  2132  func (pc *persistConn) markReused() {
  2133  	pc.mu.Lock()
  2134  	pc.reused = true
  2135  	pc.mu.Unlock()
  2136  }
  2137  
  2138  // close closes the underlying TCP connection and closes
  2139  // the pc.closech channel.
  2140  //
  2141  // The provided err is only for testing and debugging; in normal
  2142  // circumstances it should never be seen by users.
  2143  func (pc *persistConn) close(err error) {
  2144  	pc.mu.Lock()
  2145  	defer pc.mu.Unlock()
  2146  	pc.closeLocked(err)
  2147  }
  2148  
  2149  func (pc *persistConn) closeLocked(err error) {
  2150  	if err == nil {
  2151  		panic("nil error")
  2152  	}
  2153  	pc.broken = true
  2154  	if pc.closed == nil {
  2155  		pc.closed = err
  2156  		if pc.alt != nil {
  2157  			// Do nothing; can only get here via getConn's
  2158  			// handlePendingDial's putOrCloseIdleConn when
  2159  			// it turns out the abandoned connection in
  2160  			// flight ended up negotiating an alternate
  2161  			// protocol. We don't use the connection
  2162  			// freelist for http2. That's done by the
  2163  			// alternate protocol's RoundTripper.
  2164  		} else {
  2165  			pc.conn.Close()
  2166  			close(pc.closech)
  2167  		}
  2168  	}
  2169  	pc.mutateHeaderFunc = nil
  2170  }
  2171  
  2172  var portMap = map[string]string{
  2173  	"http":   "80",
  2174  	"https":  "443",
  2175  	"socks5": "1080",
  2176  }
  2177  
  2178  // canonicalAddr returns url.Host but always with a ":port" suffix
  2179  func canonicalAddr(url *url.URL) string {
  2180  	addr := url.Hostname()
  2181  	if v, err := idnaASCII(addr); err == nil {
  2182  		addr = v
  2183  	}
  2184  	port := url.Port()
  2185  	if port == "" {
  2186  		port = portMap[url.Scheme]
  2187  	}
  2188  	return net.JoinHostPort(addr, port)
  2189  }
  2190  
  2191  // bodyEOFSignal is used by the HTTP/1 transport when reading response
  2192  // bodies to make sure we see the end of a response body before
  2193  // proceeding and reading on the connection again.
  2194  //
  2195  // It wraps a ReadCloser but runs fn (if non-nil) at most
  2196  // once, right before its final (error-producing) Read or Close call
  2197  // returns. fn should return the new error to return from Read or Close.
  2198  //
  2199  // If earlyCloseFn is non-nil and Close is called before io.EOF is
  2200  // seen, earlyCloseFn is called instead of fn, and its return value is
  2201  // the return value from Close.
  2202  type bodyEOFSignal struct {
  2203  	body         io.ReadCloser
  2204  	mu           sync.Mutex        // guards following 4 fields
  2205  	closed       bool              // whether Close has been called
  2206  	rerr         error             // sticky Read error
  2207  	fn           func(error) error // err will be nil on Read io.EOF
  2208  	earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen
  2209  }
  2210  
  2211  var errReadOnClosedResBody = errors.New("http: read on closed response body")
  2212  
  2213  func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
  2214  	es.mu.Lock()
  2215  	closed, rerr := es.closed, es.rerr
  2216  	es.mu.Unlock()
  2217  	if closed {
  2218  		return 0, errReadOnClosedResBody
  2219  	}
  2220  	if rerr != nil {
  2221  		return 0, rerr
  2222  	}
  2223  
  2224  	n, err = es.body.Read(p)
  2225  	if err != nil {
  2226  		es.mu.Lock()
  2227  		defer es.mu.Unlock()
  2228  		if es.rerr == nil {
  2229  			es.rerr = err
  2230  		}
  2231  		err = es.condfn(err)
  2232  	}
  2233  	return
  2234  }
  2235  
  2236  func (es *bodyEOFSignal) Close() error {
  2237  	es.mu.Lock()
  2238  	defer es.mu.Unlock()
  2239  	if es.closed {
  2240  		return nil
  2241  	}
  2242  	es.closed = true
  2243  	if es.earlyCloseFn != nil && es.rerr != io.EOF {
  2244  		return es.earlyCloseFn()
  2245  	}
  2246  	err := es.body.Close()
  2247  	return es.condfn(err)
  2248  }
  2249  
  2250  // caller must hold es.mu.
  2251  func (es *bodyEOFSignal) condfn(err error) error {
  2252  	if es.fn == nil {
  2253  		return err
  2254  	}
  2255  	err = es.fn(err)
  2256  	es.fn = nil
  2257  	return err
  2258  }
  2259  
  2260  // gzipReader wraps a response body so it can lazily
  2261  // call gzip.NewReader on the first call to Read
  2262  type gzipReader struct {
  2263  	body *bodyEOFSignal // underlying HTTP/1 response body framing
  2264  	zr   *gzip.Reader   // lazily-initialized gzip reader
  2265  	zerr error          // any error from gzip.NewReader; sticky
  2266  }
  2267  
  2268  func (gz *gzipReader) Read(p []byte) (n int, err error) {
  2269  	if gz.zr == nil {
  2270  		if gz.zerr == nil {
  2271  			gz.zr, gz.zerr = gzip.NewReader(gz.body)
  2272  		}
  2273  		if gz.zerr != nil {
  2274  			return 0, gz.zerr
  2275  		}
  2276  	}
  2277  
  2278  	gz.body.mu.Lock()
  2279  	if gz.body.closed {
  2280  		err = errReadOnClosedResBody
  2281  	}
  2282  	gz.body.mu.Unlock()
  2283  
  2284  	if err != nil {
  2285  		return 0, err
  2286  	}
  2287  	return gz.zr.Read(p)
  2288  }
  2289  
  2290  func (gz *gzipReader) Close() error {
  2291  	return gz.body.Close()
  2292  }
  2293  
  2294  type readerAndCloser struct {
  2295  	io.Reader
  2296  	io.Closer
  2297  }
  2298  
  2299  type tlsHandshakeTimeoutError struct{}
  2300  
  2301  func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
  2302  func (tlsHandshakeTimeoutError) Temporary() bool { return true }
  2303  func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
  2304  
  2305  // fakeLocker is a sync.Locker which does nothing. It's used to guard
  2306  // test-only fields when not under test, to avoid runtime atomic
  2307  // overhead.
  2308  type fakeLocker struct{}
  2309  
  2310  func (fakeLocker) Lock()   {}
  2311  func (fakeLocker) Unlock() {}
  2312  
  2313  // clneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if
  2314  // cfg is nil. This is safe to call even if cfg is in active use by a TLS
  2315  // client or server.
  2316  func cloneTLSConfig(cfg *tls.Config) *tls.Config {
  2317  	if cfg == nil {
  2318  		return &tls.Config{}
  2319  	}
  2320  	return cfg.Clone()
  2321  }
  2322  
  2323  type connLRU struct {
  2324  	ll *list.List // list.Element.Value type is of *persistConn
  2325  	m  map[*persistConn]*list.Element
  2326  }
  2327  
  2328  // add adds pc to the head of the linked list.
  2329  func (cl *connLRU) add(pc *persistConn) {
  2330  	if cl.ll == nil {
  2331  		cl.ll = list.New()
  2332  		cl.m = make(map[*persistConn]*list.Element)
  2333  	}
  2334  	ele := cl.ll.PushFront(pc)
  2335  	if _, ok := cl.m[pc]; ok {
  2336  		panic("persistConn was already in LRU")
  2337  	}
  2338  	cl.m[pc] = ele
  2339  }
  2340  
  2341  func (cl *connLRU) removeOldest() *persistConn {
  2342  	ele := cl.ll.Back()
  2343  	pc := ele.Value.(*persistConn)
  2344  	cl.ll.Remove(ele)
  2345  	delete(cl.m, pc)
  2346  	return pc
  2347  }
  2348  
  2349  // remove removes pc from cl.
  2350  func (cl *connLRU) remove(pc *persistConn) {
  2351  	if ele, ok := cl.m[pc]; ok {
  2352  		cl.ll.Remove(ele)
  2353  		delete(cl.m, pc)
  2354  	}
  2355  }
  2356  
  2357  // len returns the number of items in the cache.
  2358  func (cl *connLRU) len() int {
  2359  	return len(cl.m)
  2360  }
  2361  
  2362  // validPort reports whether p (without the colon) is a valid port in
  2363  // a URL, per RFC 3986 Section 3.2.3, which says the port may be
  2364  // empty, or only contain digits.
  2365  func validPort(p string) bool {
  2366  	for _, r := range []byte(p) {
  2367  		if r < '0' || r > '9' {
  2368  			return false
  2369  		}
  2370  	}
  2371  	return true
  2372  }