github.com/sean-/go@v0.0.0-20151219100004-97f854cd7bb6/src/net/http/transport.go (about)

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // HTTP client implementation. See RFC 2616.
     6  //
     7  // This is the low-level Transport implementation of RoundTripper.
     8  // The high-level interface is in client.go.
     9  
    10  package http
    11  
    12  import (
    13  	"bufio"
    14  	"compress/gzip"
    15  	"crypto/tls"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"log"
    20  	"net"
    21  	"net/url"
    22  	"os"
    23  	"strings"
    24  	"sync"
    25  	"time"
    26  )
    27  
    28  // DefaultTransport is the default implementation of Transport and is
    29  // used by DefaultClient. It establishes network connections as needed
    30  // and caches them for reuse by subsequent calls. It uses HTTP proxies
    31  // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
    32  // $no_proxy) environment variables.
    33  var DefaultTransport RoundTripper = &Transport{
    34  	Proxy: ProxyFromEnvironment,
    35  	Dial: (&net.Dialer{
    36  		Timeout:   30 * time.Second,
    37  		KeepAlive: 30 * time.Second,
    38  	}).Dial,
    39  	TLSHandshakeTimeout:   10 * time.Second,
    40  	ExpectContinueTimeout: 1 * time.Second,
    41  }
    42  
    43  // DefaultMaxIdleConnsPerHost is the default value of Transport's
    44  // MaxIdleConnsPerHost.
    45  const DefaultMaxIdleConnsPerHost = 2
    46  
    47  // Transport is an implementation of RoundTripper that supports HTTP,
    48  // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
    49  // Transport can also cache connections for future re-use.
    50  type Transport struct {
    51  	idleMu     sync.Mutex
    52  	wantIdle   bool // user has requested to close all idle conns
    53  	idleConn   map[connectMethodKey][]*persistConn
    54  	idleConnCh map[connectMethodKey]chan *persistConn
    55  
    56  	reqMu       sync.Mutex
    57  	reqCanceler map[*Request]func()
    58  
    59  	altMu    sync.RWMutex
    60  	altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper
    61  
    62  	// Proxy specifies a function to return a proxy for a given
    63  	// Request. If the function returns a non-nil error, the
    64  	// request is aborted with the provided error.
    65  	// If Proxy is nil or returns a nil *URL, no proxy is used.
    66  	Proxy func(*Request) (*url.URL, error)
    67  
    68  	// Dial specifies the dial function for creating unencrypted
    69  	// TCP connections.
    70  	// If Dial is nil, net.Dial is used.
    71  	Dial func(network, addr string) (net.Conn, error)
    72  
    73  	// DialTLS specifies an optional dial function for creating
    74  	// TLS connections for non-proxied HTTPS requests.
    75  	//
    76  	// If DialTLS is nil, Dial and TLSClientConfig are used.
    77  	//
    78  	// If DialTLS is set, the Dial hook is not used for HTTPS
    79  	// requests and the TLSClientConfig and TLSHandshakeTimeout
    80  	// are ignored. The returned net.Conn is assumed to already be
    81  	// past the TLS handshake.
    82  	DialTLS func(network, addr string) (net.Conn, error)
    83  
    84  	// TLSClientConfig specifies the TLS configuration to use with
    85  	// tls.Client. If nil, the default configuration is used.
    86  	TLSClientConfig *tls.Config
    87  
    88  	// TLSHandshakeTimeout specifies the maximum amount of time waiting to
    89  	// wait for a TLS handshake. Zero means no timeout.
    90  	TLSHandshakeTimeout time.Duration
    91  
    92  	// DisableKeepAlives, if true, prevents re-use of TCP connections
    93  	// between different HTTP requests.
    94  	DisableKeepAlives bool
    95  
    96  	// DisableCompression, if true, prevents the Transport from
    97  	// requesting compression with an "Accept-Encoding: gzip"
    98  	// request header when the Request contains no existing
    99  	// Accept-Encoding value. If the Transport requests gzip on
   100  	// its own and gets a gzipped response, it's transparently
   101  	// decoded in the Response.Body. However, if the user
   102  	// explicitly requested gzip it is not automatically
   103  	// uncompressed.
   104  	DisableCompression bool
   105  
   106  	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
   107  	// (keep-alive) to keep per-host.  If zero,
   108  	// DefaultMaxIdleConnsPerHost is used.
   109  	MaxIdleConnsPerHost int
   110  
   111  	// ResponseHeaderTimeout, if non-zero, specifies the amount of
   112  	// time to wait for a server's response headers after fully
   113  	// writing the request (including its body, if any). This
   114  	// time does not include the time to read the response body.
   115  	ResponseHeaderTimeout time.Duration
   116  
   117  	// ExpectContinueTimeout, if non-zero, specifies the amount of
   118  	// time to wait for a server's first response headers after fully
   119  	// writing the request headers if the request has an
   120  	// "Expect: 100-continue" header. Zero means no timeout.
   121  	// This time does not include the time to send the request header.
   122  	ExpectContinueTimeout time.Duration
   123  
   124  	// TLSNextProto specifies how the Transport switches to an
   125  	// alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
   126  	// protocol negotiation.  If Transport dials an TLS connection
   127  	// with a non-empty protocol name and TLSNextProto contains a
   128  	// map entry for that key (such as "h2"), then the func is
   129  	// called with the request's authority (such as "example.com"
   130  	// or "example.com:1234") and the TLS connection. The function
   131  	// must return a RoundTripper that then handles the request.
   132  	// If TLSNextProto is nil, HTTP/2 support is enabled automatically.
   133  	TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper
   134  
   135  	nextProtoOnce sync.Once // guards initialization of TLSNextProto (onceSetNextProtoDefaults)
   136  
   137  	// TODO: tunable on global max cached connections
   138  	// TODO: tunable on timeout on cached connections
   139  }
   140  
   141  // onceSetNextProtoDefaults initializes TLSNextProto.
   142  // It must be called via t.nextProtoOnce.Do.
   143  func (t *Transport) onceSetNextProtoDefaults() {
   144  	if strings.Contains(os.Getenv("GODEBUG"), "h2client=0") {
   145  		return
   146  	}
   147  	if t.TLSNextProto != nil {
   148  		return
   149  	}
   150  	err := http2ConfigureTransport(t)
   151  	if err != nil {
   152  		log.Printf("Error enabling Transport HTTP/2 support: %v", err)
   153  	}
   154  }
   155  
   156  // ProxyFromEnvironment returns the URL of the proxy to use for a
   157  // given request, as indicated by the environment variables
   158  // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
   159  // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
   160  // requests.
   161  //
   162  // The environment values may be either a complete URL or a
   163  // "host[:port]", in which case the "http" scheme is assumed.
   164  // An error is returned if the value is a different form.
   165  //
   166  // A nil URL and nil error are returned if no proxy is defined in the
   167  // environment, or a proxy should not be used for the given request,
   168  // as defined by NO_PROXY.
   169  //
   170  // As a special case, if req.URL.Host is "localhost" (with or without
   171  // a port number), then a nil URL and nil error will be returned.
   172  func ProxyFromEnvironment(req *Request) (*url.URL, error) {
   173  	var proxy string
   174  	if req.URL.Scheme == "https" {
   175  		proxy = httpsProxyEnv.Get()
   176  	}
   177  	if proxy == "" {
   178  		proxy = httpProxyEnv.Get()
   179  	}
   180  	if proxy == "" {
   181  		return nil, nil
   182  	}
   183  	if !useProxy(canonicalAddr(req.URL)) {
   184  		return nil, nil
   185  	}
   186  	proxyURL, err := url.Parse(proxy)
   187  	if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
   188  		// proxy was bogus. Try prepending "http://" to it and
   189  		// see if that parses correctly. If not, we fall
   190  		// through and complain about the original one.
   191  		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
   192  			return proxyURL, nil
   193  		}
   194  	}
   195  	if err != nil {
   196  		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
   197  	}
   198  	return proxyURL, nil
   199  }
   200  
   201  // ProxyURL returns a proxy function (for use in a Transport)
   202  // that always returns the same URL.
   203  func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
   204  	return func(*Request) (*url.URL, error) {
   205  		return fixedURL, nil
   206  	}
   207  }
   208  
   209  // transportRequest is a wrapper around a *Request that adds
   210  // optional extra headers to write.
   211  type transportRequest struct {
   212  	*Request        // original request, not to be mutated
   213  	extra    Header // extra headers to write, or nil
   214  }
   215  
   216  func (tr *transportRequest) extraHeaders() Header {
   217  	if tr.extra == nil {
   218  		tr.extra = make(Header)
   219  	}
   220  	return tr.extra
   221  }
   222  
   223  // RoundTrip implements the RoundTripper interface.
   224  //
   225  // For higher-level HTTP client support (such as handling of cookies
   226  // and redirects), see Get, Post, and the Client type.
   227  func (t *Transport) RoundTrip(req *Request) (*Response, error) {
   228  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   229  	if req.URL == nil {
   230  		req.closeBody()
   231  		return nil, errors.New("http: nil Request.URL")
   232  	}
   233  	if req.Header == nil {
   234  		req.closeBody()
   235  		return nil, errors.New("http: nil Request.Header")
   236  	}
   237  	// TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex
   238  	t.altMu.RLock()
   239  	altRT := t.altProto[req.URL.Scheme]
   240  	t.altMu.RUnlock()
   241  	if altRT != nil {
   242  		if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
   243  			return resp, err
   244  		}
   245  	}
   246  	if s := req.URL.Scheme; s != "http" && s != "https" {
   247  		req.closeBody()
   248  		return nil, &badStringError{"unsupported protocol scheme", s}
   249  	}
   250  	if req.Method != "" && !validMethod(req.Method) {
   251  		return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
   252  	}
   253  	if req.URL.Host == "" {
   254  		req.closeBody()
   255  		return nil, errors.New("http: no Host in request URL")
   256  	}
   257  
   258  	for {
   259  		// treq gets modified by roundTrip, so we need to recreate for each retry.
   260  		treq := &transportRequest{Request: req}
   261  		cm, err := t.connectMethodForRequest(treq)
   262  		if err != nil {
   263  			req.closeBody()
   264  			return nil, err
   265  		}
   266  
   267  		// Get the cached or newly-created connection to either the
   268  		// host (for http or https), the http proxy, or the http proxy
   269  		// pre-CONNECTed to https server.  In any case, we'll be ready
   270  		// to send it requests.
   271  		pconn, err := t.getConn(req, cm)
   272  		if err != nil {
   273  			t.setReqCanceler(req, nil)
   274  			req.closeBody()
   275  			return nil, err
   276  		}
   277  
   278  		var resp *Response
   279  		if pconn.alt != nil {
   280  			// HTTP/2 path.
   281  			resp, err = pconn.alt.RoundTrip(req)
   282  		} else {
   283  			resp, err = pconn.roundTrip(treq)
   284  		}
   285  		if err == nil {
   286  			return resp, nil
   287  		}
   288  		if err := checkTransportResend(err, req, pconn); err != nil {
   289  			return nil, err
   290  		}
   291  		testHookRoundTripRetried()
   292  	}
   293  }
   294  
   295  // checkTransportResend checks whether a failed HTTP request can be
   296  // resent on a new connection. The non-nil input error is the error from
   297  // roundTrip, which might be wrapped in a beforeRespHeaderError error.
   298  //
   299  // The return value is err or the unwrapped error inside a
   300  // beforeRespHeaderError.
   301  func checkTransportResend(err error, req *Request, pconn *persistConn) error {
   302  	brhErr, ok := err.(beforeRespHeaderError)
   303  	if !ok {
   304  		return err
   305  	}
   306  	err = brhErr.error // unwrap the custom error in case we return it
   307  	if err != errMissingHost && pconn.isReused() && req.isReplayable() {
   308  		// If we try to reuse a connection that the server is in the process of
   309  		// closing, we may end up successfully writing out our request (or a
   310  		// portion of our request) only to find a connection error when we try to
   311  		// read from (or finish writing to) the socket.
   312  
   313  		// There can be a race between the socket pool checking whether a socket
   314  		// is still connected, receiving the FIN, and sending/reading data on a
   315  		// reused socket. If we receive the FIN between the connectedness check
   316  		// and writing/reading from the socket, we may first learn the socket is
   317  		// disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most
   318  		// likely happen when trying to retrieve its IP address. See
   319  		// http://crbug.com/105824 for more details.
   320  
   321  		// We resend a request only if we reused a keep-alive connection and did
   322  		// not yet receive any header data. This automatically prevents an
   323  		// infinite resend loop because we'll run out of the cached keep-alive
   324  		// connections eventually.
   325  		return nil
   326  	}
   327  	return err
   328  }
   329  
   330  // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
   331  var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")
   332  
   333  // RegisterProtocol registers a new protocol with scheme.
   334  // The Transport will pass requests using the given scheme to rt.
   335  // It is rt's responsibility to simulate HTTP request semantics.
   336  //
   337  // RegisterProtocol can be used by other packages to provide
   338  // implementations of protocol schemes like "ftp" or "file".
   339  //
   340  // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
   341  // handle the RoundTrip itself for that one request, as if the
   342  // protocol were not registered.
   343  func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
   344  	t.altMu.Lock()
   345  	defer t.altMu.Unlock()
   346  	if t.altProto == nil {
   347  		t.altProto = make(map[string]RoundTripper)
   348  	}
   349  	if _, exists := t.altProto[scheme]; exists {
   350  		panic("protocol " + scheme + " already registered")
   351  	}
   352  	t.altProto[scheme] = rt
   353  }
   354  
   355  // CloseIdleConnections closes any connections which were previously
   356  // connected from previous requests but are now sitting idle in
   357  // a "keep-alive" state. It does not interrupt any connections currently
   358  // in use.
   359  func (t *Transport) CloseIdleConnections() {
   360  	t.idleMu.Lock()
   361  	m := t.idleConn
   362  	t.idleConn = nil
   363  	t.idleConnCh = nil
   364  	t.wantIdle = true
   365  	t.idleMu.Unlock()
   366  	for _, conns := range m {
   367  		for _, pconn := range conns {
   368  			pconn.close()
   369  		}
   370  	}
   371  }
   372  
   373  // CancelRequest cancels an in-flight request by closing its connection.
   374  // CancelRequest should only be called after RoundTrip has returned.
   375  func (t *Transport) CancelRequest(req *Request) {
   376  	t.reqMu.Lock()
   377  	cancel := t.reqCanceler[req]
   378  	delete(t.reqCanceler, req)
   379  	t.reqMu.Unlock()
   380  	if cancel != nil {
   381  		cancel()
   382  	}
   383  }
   384  
   385  //
   386  // Private implementation past this point.
   387  //
   388  
   389  var (
   390  	httpProxyEnv = &envOnce{
   391  		names: []string{"HTTP_PROXY", "http_proxy"},
   392  	}
   393  	httpsProxyEnv = &envOnce{
   394  		names: []string{"HTTPS_PROXY", "https_proxy"},
   395  	}
   396  	noProxyEnv = &envOnce{
   397  		names: []string{"NO_PROXY", "no_proxy"},
   398  	}
   399  )
   400  
   401  // envOnce looks up an environment variable (optionally by multiple
   402  // names) once. It mitigates expensive lookups on some platforms
   403  // (e.g. Windows).
   404  type envOnce struct {
   405  	names []string
   406  	once  sync.Once
   407  	val   string
   408  }
   409  
   410  func (e *envOnce) Get() string {
   411  	e.once.Do(e.init)
   412  	return e.val
   413  }
   414  
   415  func (e *envOnce) init() {
   416  	for _, n := range e.names {
   417  		e.val = os.Getenv(n)
   418  		if e.val != "" {
   419  			return
   420  		}
   421  	}
   422  }
   423  
   424  // reset is used by tests
   425  func (e *envOnce) reset() {
   426  	e.once = sync.Once{}
   427  	e.val = ""
   428  }
   429  
   430  func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
   431  	cm.targetScheme = treq.URL.Scheme
   432  	cm.targetAddr = canonicalAddr(treq.URL)
   433  	if t.Proxy != nil {
   434  		cm.proxyURL, err = t.Proxy(treq.Request)
   435  	}
   436  	return cm, err
   437  }
   438  
   439  // proxyAuth returns the Proxy-Authorization header to set
   440  // on requests, if applicable.
   441  func (cm *connectMethod) proxyAuth() string {
   442  	if cm.proxyURL == nil {
   443  		return ""
   444  	}
   445  	if u := cm.proxyURL.User; u != nil {
   446  		username := u.Username()
   447  		password, _ := u.Password()
   448  		return "Basic " + basicAuth(username, password)
   449  	}
   450  	return ""
   451  }
   452  
   453  // putIdleConn adds pconn to the list of idle persistent connections awaiting
   454  // a new request.
   455  // If pconn is no longer needed or not in a good state, putIdleConn
   456  // returns false.
   457  func (t *Transport) putIdleConn(pconn *persistConn) bool {
   458  	if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
   459  		pconn.close()
   460  		return false
   461  	}
   462  	if pconn.isBroken() {
   463  		return false
   464  	}
   465  	key := pconn.cacheKey
   466  	max := t.MaxIdleConnsPerHost
   467  	if max == 0 {
   468  		max = DefaultMaxIdleConnsPerHost
   469  	}
   470  	pconn.markReused()
   471  	t.idleMu.Lock()
   472  
   473  	waitingDialer := t.idleConnCh[key]
   474  	select {
   475  	case waitingDialer <- pconn:
   476  		// We're done with this pconn and somebody else is
   477  		// currently waiting for a conn of this type (they're
   478  		// actively dialing, but this conn is ready
   479  		// first). Chrome calls this socket late binding.  See
   480  		// https://insouciant.org/tech/connection-management-in-chromium/
   481  		t.idleMu.Unlock()
   482  		return true
   483  	default:
   484  		if waitingDialer != nil {
   485  			// They had populated this, but their dial won
   486  			// first, so we can clean up this map entry.
   487  			delete(t.idleConnCh, key)
   488  		}
   489  	}
   490  	if t.wantIdle {
   491  		t.idleMu.Unlock()
   492  		pconn.close()
   493  		return false
   494  	}
   495  	if t.idleConn == nil {
   496  		t.idleConn = make(map[connectMethodKey][]*persistConn)
   497  	}
   498  	if len(t.idleConn[key]) >= max {
   499  		t.idleMu.Unlock()
   500  		pconn.close()
   501  		return false
   502  	}
   503  	for _, exist := range t.idleConn[key] {
   504  		if exist == pconn {
   505  			log.Fatalf("dup idle pconn %p in freelist", pconn)
   506  		}
   507  	}
   508  	t.idleConn[key] = append(t.idleConn[key], pconn)
   509  	t.idleMu.Unlock()
   510  	return true
   511  }
   512  
   513  // getIdleConnCh returns a channel to receive and return idle
   514  // persistent connection for the given connectMethod.
   515  // It may return nil, if persistent connections are not being used.
   516  func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn {
   517  	if t.DisableKeepAlives {
   518  		return nil
   519  	}
   520  	key := cm.key()
   521  	t.idleMu.Lock()
   522  	defer t.idleMu.Unlock()
   523  	t.wantIdle = false
   524  	if t.idleConnCh == nil {
   525  		t.idleConnCh = make(map[connectMethodKey]chan *persistConn)
   526  	}
   527  	ch, ok := t.idleConnCh[key]
   528  	if !ok {
   529  		ch = make(chan *persistConn)
   530  		t.idleConnCh[key] = ch
   531  	}
   532  	return ch
   533  }
   534  
   535  func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) {
   536  	key := cm.key()
   537  	t.idleMu.Lock()
   538  	defer t.idleMu.Unlock()
   539  	if t.idleConn == nil {
   540  		return nil
   541  	}
   542  	for {
   543  		pconns, ok := t.idleConn[key]
   544  		if !ok {
   545  			return nil
   546  		}
   547  		if len(pconns) == 1 {
   548  			pconn = pconns[0]
   549  			delete(t.idleConn, key)
   550  		} else {
   551  			// 2 or more cached connections; pop last
   552  			// TODO: queue?
   553  			pconn = pconns[len(pconns)-1]
   554  			t.idleConn[key] = pconns[:len(pconns)-1]
   555  		}
   556  		if !pconn.isBroken() {
   557  			return
   558  		}
   559  	}
   560  }
   561  
   562  func (t *Transport) setReqCanceler(r *Request, fn func()) {
   563  	t.reqMu.Lock()
   564  	defer t.reqMu.Unlock()
   565  	if t.reqCanceler == nil {
   566  		t.reqCanceler = make(map[*Request]func())
   567  	}
   568  	if fn != nil {
   569  		t.reqCanceler[r] = fn
   570  	} else {
   571  		delete(t.reqCanceler, r)
   572  	}
   573  }
   574  
   575  // replaceReqCanceler replaces an existing cancel function. If there is no cancel function
   576  // for the request, we don't set the function and return false.
   577  // Since CancelRequest will clear the canceler, we can use the return value to detect if
   578  // the request was canceled since the last setReqCancel call.
   579  func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool {
   580  	t.reqMu.Lock()
   581  	defer t.reqMu.Unlock()
   582  	_, ok := t.reqCanceler[r]
   583  	if !ok {
   584  		return false
   585  	}
   586  	if fn != nil {
   587  		t.reqCanceler[r] = fn
   588  	} else {
   589  		delete(t.reqCanceler, r)
   590  	}
   591  	return true
   592  }
   593  
   594  func (t *Transport) dial(network, addr string) (c net.Conn, err error) {
   595  	if t.Dial != nil {
   596  		return t.Dial(network, addr)
   597  	}
   598  	return net.Dial(network, addr)
   599  }
   600  
   601  // getConn dials and creates a new persistConn to the target as
   602  // specified in the connectMethod.  This includes doing a proxy CONNECT
   603  // and/or setting up TLS.  If this doesn't return an error, the persistConn
   604  // is ready to write requests to.
   605  func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) {
   606  	if pc := t.getIdleConn(cm); pc != nil {
   607  		// set request canceler to some non-nil function so we
   608  		// can detect whether it was cleared between now and when
   609  		// we enter roundTrip
   610  		t.setReqCanceler(req, func() {})
   611  		return pc, nil
   612  	}
   613  
   614  	type dialRes struct {
   615  		pc  *persistConn
   616  		err error
   617  	}
   618  	dialc := make(chan dialRes)
   619  
   620  	// Copy these hooks so we don't race on the postPendingDial in
   621  	// the goroutine we launch. Issue 11136.
   622  	testHookPrePendingDial := testHookPrePendingDial
   623  	testHookPostPendingDial := testHookPostPendingDial
   624  
   625  	handlePendingDial := func() {
   626  		testHookPrePendingDial()
   627  		go func() {
   628  			if v := <-dialc; v.err == nil {
   629  				t.putIdleConn(v.pc)
   630  			}
   631  			testHookPostPendingDial()
   632  		}()
   633  	}
   634  
   635  	cancelc := make(chan struct{})
   636  	t.setReqCanceler(req, func() { close(cancelc) })
   637  
   638  	go func() {
   639  		pc, err := t.dialConn(cm)
   640  		dialc <- dialRes{pc, err}
   641  	}()
   642  
   643  	idleConnCh := t.getIdleConnCh(cm)
   644  	select {
   645  	case v := <-dialc:
   646  		// Our dial finished.
   647  		return v.pc, v.err
   648  	case pc := <-idleConnCh:
   649  		// Another request finished first and its net.Conn
   650  		// became available before our dial. Or somebody
   651  		// else's dial that they didn't use.
   652  		// But our dial is still going, so give it away
   653  		// when it finishes:
   654  		handlePendingDial()
   655  		return pc, nil
   656  	case <-req.Cancel:
   657  		handlePendingDial()
   658  		return nil, errors.New("net/http: request canceled while waiting for connection")
   659  	case <-cancelc:
   660  		handlePendingDial()
   661  		return nil, errors.New("net/http: request canceled while waiting for connection")
   662  	}
   663  }
   664  
   665  func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) {
   666  	pconn := &persistConn{
   667  		t:          t,
   668  		cacheKey:   cm.key(),
   669  		reqch:      make(chan requestAndChan, 1),
   670  		writech:    make(chan writeRequest, 1),
   671  		closech:    make(chan struct{}),
   672  		writeErrCh: make(chan error, 1),
   673  	}
   674  	tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil
   675  	if tlsDial {
   676  		var err error
   677  		pconn.conn, err = t.DialTLS("tcp", cm.addr())
   678  		if err != nil {
   679  			return nil, err
   680  		}
   681  		if tc, ok := pconn.conn.(*tls.Conn); ok {
   682  			cs := tc.ConnectionState()
   683  			pconn.tlsState = &cs
   684  		}
   685  	} else {
   686  		conn, err := t.dial("tcp", cm.addr())
   687  		if err != nil {
   688  			if cm.proxyURL != nil {
   689  				err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
   690  			}
   691  			return nil, err
   692  		}
   693  		pconn.conn = conn
   694  	}
   695  
   696  	// Proxy setup.
   697  	switch {
   698  	case cm.proxyURL == nil:
   699  		// Do nothing. Not using a proxy.
   700  	case cm.targetScheme == "http":
   701  		pconn.isProxy = true
   702  		if pa := cm.proxyAuth(); pa != "" {
   703  			pconn.mutateHeaderFunc = func(h Header) {
   704  				h.Set("Proxy-Authorization", pa)
   705  			}
   706  		}
   707  	case cm.targetScheme == "https":
   708  		conn := pconn.conn
   709  		connectReq := &Request{
   710  			Method: "CONNECT",
   711  			URL:    &url.URL{Opaque: cm.targetAddr},
   712  			Host:   cm.targetAddr,
   713  			Header: make(Header),
   714  		}
   715  		if pa := cm.proxyAuth(); pa != "" {
   716  			connectReq.Header.Set("Proxy-Authorization", pa)
   717  		}
   718  		connectReq.Write(conn)
   719  
   720  		// Read response.
   721  		// Okay to use and discard buffered reader here, because
   722  		// TLS server will not speak until spoken to.
   723  		br := bufio.NewReader(conn)
   724  		resp, err := ReadResponse(br, connectReq)
   725  		if err != nil {
   726  			conn.Close()
   727  			return nil, err
   728  		}
   729  		if resp.StatusCode != 200 {
   730  			f := strings.SplitN(resp.Status, " ", 2)
   731  			conn.Close()
   732  			return nil, errors.New(f[1])
   733  		}
   734  	}
   735  
   736  	if cm.targetScheme == "https" && !tlsDial {
   737  		// Initiate TLS and check remote host name against certificate.
   738  		cfg := cloneTLSClientConfig(t.TLSClientConfig)
   739  		if cfg.ServerName == "" {
   740  			cfg.ServerName = cm.tlsHost()
   741  		}
   742  		plainConn := pconn.conn
   743  		tlsConn := tls.Client(plainConn, cfg)
   744  		errc := make(chan error, 2)
   745  		var timer *time.Timer // for canceling TLS handshake
   746  		if d := t.TLSHandshakeTimeout; d != 0 {
   747  			timer = time.AfterFunc(d, func() {
   748  				errc <- tlsHandshakeTimeoutError{}
   749  			})
   750  		}
   751  		go func() {
   752  			err := tlsConn.Handshake()
   753  			if timer != nil {
   754  				timer.Stop()
   755  			}
   756  			errc <- err
   757  		}()
   758  		if err := <-errc; err != nil {
   759  			plainConn.Close()
   760  			return nil, err
   761  		}
   762  		if !cfg.InsecureSkipVerify {
   763  			if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
   764  				plainConn.Close()
   765  				return nil, err
   766  			}
   767  		}
   768  		cs := tlsConn.ConnectionState()
   769  		pconn.tlsState = &cs
   770  		pconn.conn = tlsConn
   771  	}
   772  
   773  	if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
   774  		if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
   775  			return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil
   776  		}
   777  	}
   778  
   779  	pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF})
   780  	pconn.bw = bufio.NewWriter(pconn.conn)
   781  	go pconn.readLoop()
   782  	go pconn.writeLoop()
   783  	return pconn, nil
   784  }
   785  
   786  // useProxy reports whether requests to addr should use a proxy,
   787  // according to the NO_PROXY or no_proxy environment variable.
   788  // addr is always a canonicalAddr with a host and port.
   789  func useProxy(addr string) bool {
   790  	if len(addr) == 0 {
   791  		return true
   792  	}
   793  	host, _, err := net.SplitHostPort(addr)
   794  	if err != nil {
   795  		return false
   796  	}
   797  	if host == "localhost" {
   798  		return false
   799  	}
   800  	if ip := net.ParseIP(host); ip != nil {
   801  		if ip.IsLoopback() {
   802  			return false
   803  		}
   804  	}
   805  
   806  	no_proxy := noProxyEnv.Get()
   807  	if no_proxy == "*" {
   808  		return false
   809  	}
   810  
   811  	addr = strings.ToLower(strings.TrimSpace(addr))
   812  	if hasPort(addr) {
   813  		addr = addr[:strings.LastIndex(addr, ":")]
   814  	}
   815  
   816  	for _, p := range strings.Split(no_proxy, ",") {
   817  		p = strings.ToLower(strings.TrimSpace(p))
   818  		if len(p) == 0 {
   819  			continue
   820  		}
   821  		if hasPort(p) {
   822  			p = p[:strings.LastIndex(p, ":")]
   823  		}
   824  		if addr == p {
   825  			return false
   826  		}
   827  		if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
   828  			// no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
   829  			return false
   830  		}
   831  		if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
   832  			// no_proxy "foo.com" matches "bar.foo.com"
   833  			return false
   834  		}
   835  	}
   836  	return true
   837  }
   838  
   839  // connectMethod is the map key (in its String form) for keeping persistent
   840  // TCP connections alive for subsequent HTTP requests.
   841  //
   842  // A connect method may be of the following types:
   843  //
   844  // Cache key form                Description
   845  // -----------------             -------------------------
   846  // |http|foo.com                 http directly to server, no proxy
   847  // |https|foo.com                https directly to server, no proxy
   848  // http://proxy.com|https|foo.com  http to proxy, then CONNECT to foo.com
   849  // http://proxy.com|http           http to proxy, http to anywhere after that
   850  //
   851  // Note: no support to https to the proxy yet.
   852  //
   853  type connectMethod struct {
   854  	proxyURL     *url.URL // nil for no proxy, else full proxy URL
   855  	targetScheme string   // "http" or "https"
   856  	targetAddr   string   // Not used if proxy + http targetScheme (4th example in table)
   857  }
   858  
   859  func (cm *connectMethod) key() connectMethodKey {
   860  	proxyStr := ""
   861  	targetAddr := cm.targetAddr
   862  	if cm.proxyURL != nil {
   863  		proxyStr = cm.proxyURL.String()
   864  		if cm.targetScheme == "http" {
   865  			targetAddr = ""
   866  		}
   867  	}
   868  	return connectMethodKey{
   869  		proxy:  proxyStr,
   870  		scheme: cm.targetScheme,
   871  		addr:   targetAddr,
   872  	}
   873  }
   874  
   875  // addr returns the first hop "host:port" to which we need to TCP connect.
   876  func (cm *connectMethod) addr() string {
   877  	if cm.proxyURL != nil {
   878  		return canonicalAddr(cm.proxyURL)
   879  	}
   880  	return cm.targetAddr
   881  }
   882  
   883  // tlsHost returns the host name to match against the peer's
   884  // TLS certificate.
   885  func (cm *connectMethod) tlsHost() string {
   886  	h := cm.targetAddr
   887  	if hasPort(h) {
   888  		h = h[:strings.LastIndex(h, ":")]
   889  	}
   890  	return h
   891  }
   892  
   893  // connectMethodKey is the map key version of connectMethod, with a
   894  // stringified proxy URL (or the empty string) instead of a pointer to
   895  // a URL.
   896  type connectMethodKey struct {
   897  	proxy, scheme, addr string
   898  }
   899  
   900  func (k connectMethodKey) String() string {
   901  	// Only used by tests.
   902  	return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr)
   903  }
   904  
   905  // persistConn wraps a connection, usually a persistent one
   906  // (but may be used for non-keep-alive requests as well)
   907  type persistConn struct {
   908  	// alt optionally specifies the TLS NextProto RoundTripper.
   909  	// This is used for HTTP/2 today and future protocol laters.
   910  	// If it's non-nil, the rest of the fields are unused.
   911  	alt RoundTripper
   912  
   913  	t        *Transport
   914  	cacheKey connectMethodKey
   915  	conn     net.Conn
   916  	tlsState *tls.ConnectionState
   917  	br       *bufio.Reader       // from conn
   918  	sawEOF   bool                // whether we've seen EOF from conn; owned by readLoop
   919  	bw       *bufio.Writer       // to conn
   920  	reqch    chan requestAndChan // written by roundTrip; read by readLoop
   921  	writech  chan writeRequest   // written by roundTrip; read by writeLoop
   922  	closech  chan struct{}       // closed when conn closed
   923  	isProxy  bool
   924  	// writeErrCh passes the request write error (usually nil)
   925  	// from the writeLoop goroutine to the readLoop which passes
   926  	// it off to the res.Body reader, which then uses it to decide
   927  	// whether or not a connection can be reused. Issue 7569.
   928  	writeErrCh chan error
   929  
   930  	lk                   sync.Mutex // guards following fields
   931  	numExpectedResponses int
   932  	closed               bool // whether conn has been closed
   933  	broken               bool // an error has happened on this connection; marked broken so it's not reused.
   934  	canceled             bool // whether this conn was broken due a CancelRequest
   935  	reused               bool // whether conn has had successful request/response and is being reused.
   936  	// mutateHeaderFunc is an optional func to modify extra
   937  	// headers on each outbound request before it's written. (the
   938  	// original Request given to RoundTrip is not modified)
   939  	mutateHeaderFunc func(Header)
   940  }
   941  
   942  // isBroken reports whether this connection is in a known broken state.
   943  func (pc *persistConn) isBroken() bool {
   944  	pc.lk.Lock()
   945  	b := pc.broken
   946  	pc.lk.Unlock()
   947  	return b
   948  }
   949  
   950  // isCanceled reports whether this connection was closed due to CancelRequest.
   951  func (pc *persistConn) isCanceled() bool {
   952  	pc.lk.Lock()
   953  	defer pc.lk.Unlock()
   954  	return pc.canceled
   955  }
   956  
   957  // isReused reports whether this connection is in a known broken state.
   958  func (pc *persistConn) isReused() bool {
   959  	pc.lk.Lock()
   960  	r := pc.reused
   961  	pc.lk.Unlock()
   962  	return r
   963  }
   964  
   965  func (pc *persistConn) cancelRequest() {
   966  	pc.lk.Lock()
   967  	defer pc.lk.Unlock()
   968  	pc.canceled = true
   969  	pc.closeLocked()
   970  }
   971  
   972  func (pc *persistConn) readLoop() {
   973  	defer pc.close()
   974  
   975  	// eofc is used to block caller goroutines reading from Response.Body
   976  	// at EOF until this goroutines has (potentially) added the connection
   977  	// back to the idle pool.
   978  	eofc := make(chan struct{})
   979  	defer close(eofc) // unblock reader on errors
   980  
   981  	// Read this once, before loop starts. (to avoid races in tests)
   982  	testHookMu.Lock()
   983  	testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
   984  	testHookMu.Unlock()
   985  
   986  	alive := true
   987  	for alive {
   988  		_, err := pc.br.Peek(1)
   989  		if err != nil {
   990  			err = beforeRespHeaderError{err}
   991  		}
   992  
   993  		pc.lk.Lock()
   994  		if pc.numExpectedResponses == 0 {
   995  			pc.readLoopPeekFailLocked(err)
   996  			pc.lk.Unlock()
   997  			return
   998  		}
   999  		pc.lk.Unlock()
  1000  
  1001  		rc := <-pc.reqch
  1002  
  1003  		var resp *Response
  1004  		if err == nil {
  1005  			resp, err = pc.readResponse(rc)
  1006  		}
  1007  
  1008  		if err != nil {
  1009  			// If we won't be able to retry this request later (from the
  1010  			// roundTrip goroutine), mark it as done now.
  1011  			// BEFORE the send on rc.ch, as the client might re-use the
  1012  			// same *Request pointer, and we don't want to set call
  1013  			// t.setReqCanceler from this persistConn while the Transport
  1014  			// potentially spins up a different persistConn for the
  1015  			// caller's subsequent request.
  1016  			if checkTransportResend(err, rc.req, pc) != nil {
  1017  				pc.t.setReqCanceler(rc.req, nil)
  1018  			}
  1019  			rc.ch <- responseAndError{err: err}
  1020  			return
  1021  		}
  1022  
  1023  		pc.lk.Lock()
  1024  		pc.numExpectedResponses--
  1025  		pc.lk.Unlock()
  1026  
  1027  		hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
  1028  
  1029  		if resp.Close || rc.req.Close || resp.StatusCode <= 199 {
  1030  			// Don't do keep-alive on error if either party requested a close
  1031  			// or we get an unexpected informational (1xx) response.
  1032  			// StatusCode 100 is already handled above.
  1033  			alive = false
  1034  		}
  1035  
  1036  		if !hasBody {
  1037  			pc.t.setReqCanceler(rc.req, nil)
  1038  			resc := make(chan *Response)        // unbuffered matters; see below
  1039  			rc.ch <- responseAndError{ch: resc} // buffered send
  1040  
  1041  			// Put the idle conn back into the pool before we send the response
  1042  			// so if they process it quickly and make another request, they'll
  1043  			// get this same conn. But we use the unbuffered channel 'rc'
  1044  			// to guarantee that persistConn.roundTrip got out of its select
  1045  			// potentially waiting for this persistConn to close.
  1046  			// but after
  1047  			alive = alive &&
  1048  				!pc.sawEOF &&
  1049  				pc.wroteRequest() &&
  1050  				pc.t.putIdleConn(pc)
  1051  
  1052  			resc <- resp // unbuffered send
  1053  
  1054  			// Now that they've read from the unbuffered channel, they're safely
  1055  			// out of the select that also waits on this goroutine to die, so
  1056  			// we're allowed to exit now if needed (if alive is false)
  1057  			testHookReadLoopBeforeNextRead()
  1058  			continue
  1059  		}
  1060  
  1061  		if rc.addedGzip {
  1062  			maybeUngzipResponse(resp)
  1063  		}
  1064  		resp.Body = &bodyEOFSignal{body: resp.Body}
  1065  
  1066  		waitForBodyRead := make(chan bool, 2)
  1067  		resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error {
  1068  			waitForBodyRead <- false
  1069  			return nil
  1070  		}
  1071  		resp.Body.(*bodyEOFSignal).fn = func(err error) error {
  1072  			isEOF := err == io.EOF
  1073  			waitForBodyRead <- isEOF
  1074  			if isEOF {
  1075  				<-eofc // see comment above eofc declaration
  1076  			} else if err != nil && pc.isCanceled() {
  1077  				return errRequestCanceled
  1078  			}
  1079  			return err
  1080  		}
  1081  
  1082  		rc.ch <- responseAndError{r: resp}
  1083  
  1084  		// Before looping back to the top of this function and peeking on
  1085  		// the bufio.Reader, wait for the caller goroutine to finish
  1086  		// reading the response body. (or for cancelation or death)
  1087  		select {
  1088  		case bodyEOF := <-waitForBodyRead:
  1089  			pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool
  1090  			alive = alive &&
  1091  				bodyEOF &&
  1092  				!pc.sawEOF &&
  1093  				pc.wroteRequest() &&
  1094  				pc.t.putIdleConn(pc)
  1095  			if bodyEOF {
  1096  				eofc <- struct{}{}
  1097  			}
  1098  		case <-rc.req.Cancel:
  1099  			alive = false
  1100  			pc.t.CancelRequest(rc.req)
  1101  		case <-pc.closech:
  1102  			alive = false
  1103  		}
  1104  
  1105  		testHookReadLoopBeforeNextRead()
  1106  	}
  1107  }
  1108  
  1109  func maybeUngzipResponse(resp *Response) {
  1110  	if resp.Header.Get("Content-Encoding") == "gzip" {
  1111  		resp.Header.Del("Content-Encoding")
  1112  		resp.Header.Del("Content-Length")
  1113  		resp.ContentLength = -1
  1114  		resp.Body = &gzipReader{body: resp.Body}
  1115  	}
  1116  }
  1117  
  1118  func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
  1119  	if pc.closed {
  1120  		return
  1121  	}
  1122  	if n := pc.br.Buffered(); n > 0 {
  1123  		buf, _ := pc.br.Peek(n)
  1124  		log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
  1125  	}
  1126  	pc.closeLocked()
  1127  }
  1128  
  1129  // readResponse reads an HTTP response (or two, in the case of "Expect:
  1130  // 100-continue") from the server. It returns the final non-100 one.
  1131  func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) {
  1132  	resp, err = ReadResponse(pc.br, rc.req)
  1133  	if err != nil {
  1134  		return
  1135  	}
  1136  	if rc.continueCh != nil {
  1137  		if resp.StatusCode == 100 {
  1138  			rc.continueCh <- struct{}{}
  1139  		} else {
  1140  			close(rc.continueCh)
  1141  		}
  1142  	}
  1143  	if resp.StatusCode == 100 {
  1144  		resp, err = ReadResponse(pc.br, rc.req)
  1145  		if err != nil {
  1146  			return
  1147  		}
  1148  	}
  1149  	resp.TLS = pc.tlsState
  1150  	return
  1151  }
  1152  
  1153  // waitForContinue returns the function to block until
  1154  // any response, timeout or connection close. After any of them,
  1155  // the function returns a bool which indicates if the body should be sent.
  1156  func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
  1157  	if continueCh == nil {
  1158  		return nil
  1159  	}
  1160  	return func() bool {
  1161  		timer := time.NewTimer(pc.t.ExpectContinueTimeout)
  1162  		defer timer.Stop()
  1163  
  1164  		select {
  1165  		case _, ok := <-continueCh:
  1166  			return ok
  1167  		case <-timer.C:
  1168  			return true
  1169  		case <-pc.closech:
  1170  			return false
  1171  		}
  1172  	}
  1173  }
  1174  
  1175  func (pc *persistConn) writeLoop() {
  1176  	for {
  1177  		select {
  1178  		case wr := <-pc.writech:
  1179  			if pc.isBroken() {
  1180  				wr.ch <- errors.New("http: can't write HTTP request on broken connection")
  1181  				continue
  1182  			}
  1183  			err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
  1184  			if err == nil {
  1185  				err = pc.bw.Flush()
  1186  			}
  1187  			if err != nil {
  1188  				pc.markBroken()
  1189  				wr.req.Request.closeBody()
  1190  			}
  1191  			pc.writeErrCh <- err // to the body reader, which might recycle us
  1192  			wr.ch <- err         // to the roundTrip function
  1193  		case <-pc.closech:
  1194  			return
  1195  		}
  1196  	}
  1197  }
  1198  
  1199  // wroteRequest is a check before recycling a connection that the previous write
  1200  // (from writeLoop above) happened and was successful.
  1201  func (pc *persistConn) wroteRequest() bool {
  1202  	select {
  1203  	case err := <-pc.writeErrCh:
  1204  		// Common case: the write happened well before the response, so
  1205  		// avoid creating a timer.
  1206  		return err == nil
  1207  	default:
  1208  		// Rare case: the request was written in writeLoop above but
  1209  		// before it could send to pc.writeErrCh, the reader read it
  1210  		// all, processed it, and called us here. In this case, give the
  1211  		// write goroutine a bit of time to finish its send.
  1212  		//
  1213  		// Less rare case: We also get here in the legitimate case of
  1214  		// Issue 7569, where the writer is still writing (or stalled),
  1215  		// but the server has already replied. In this case, we don't
  1216  		// want to wait too long, and we want to return false so this
  1217  		// connection isn't re-used.
  1218  		select {
  1219  		case err := <-pc.writeErrCh:
  1220  			return err == nil
  1221  		case <-time.After(50 * time.Millisecond):
  1222  			return false
  1223  		}
  1224  	}
  1225  }
  1226  
  1227  // responseAndError is how the goroutine reading from an HTTP/1 server
  1228  // communicates with the goroutine doing the RoundTrip.
  1229  type responseAndError struct {
  1230  	ch  chan *Response // if non-nil, res should be read from here
  1231  	r   *Response      // else use this response (see res method)
  1232  	err error
  1233  }
  1234  
  1235  func (re responseAndError) res() *Response {
  1236  	switch {
  1237  	case re.err != nil:
  1238  		return nil
  1239  	case re.ch != nil:
  1240  		return <-re.ch
  1241  	default:
  1242  		return re.r
  1243  	}
  1244  }
  1245  
  1246  type requestAndChan struct {
  1247  	req *Request
  1248  	ch  chan responseAndError
  1249  
  1250  	// did the Transport (as opposed to the client code) add an
  1251  	// Accept-Encoding gzip header? only if it we set it do
  1252  	// we transparently decode the gzip.
  1253  	addedGzip bool
  1254  
  1255  	// Optional blocking chan for Expect: 100-continue (for send).
  1256  	// If the request has an "Expect: 100-continue" header and
  1257  	// the server responds 100 Continue, readLoop send a value
  1258  	// to writeLoop via this chan.
  1259  	continueCh chan<- struct{}
  1260  }
  1261  
  1262  // A writeRequest is sent by the readLoop's goroutine to the
  1263  // writeLoop's goroutine to write a request while the read loop
  1264  // concurrently waits on both the write response and the server's
  1265  // reply.
  1266  type writeRequest struct {
  1267  	req *transportRequest
  1268  	ch  chan<- error
  1269  
  1270  	// Optional blocking chan for Expect: 100-continue (for recieve).
  1271  	// If not nil, writeLoop blocks sending request body until
  1272  	// it receives from this chan.
  1273  	continueCh <-chan struct{}
  1274  }
  1275  
  1276  type httpError struct {
  1277  	err     string
  1278  	timeout bool
  1279  }
  1280  
  1281  func (e *httpError) Error() string   { return e.err }
  1282  func (e *httpError) Timeout() bool   { return e.timeout }
  1283  func (e *httpError) Temporary() bool { return true }
  1284  
  1285  var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
  1286  var errClosed error = &httpError{err: "net/http: transport closed before response was received"}
  1287  var errRequestCanceled = errors.New("net/http: request canceled")
  1288  
  1289  func nop() {}
  1290  
  1291  // testHooks. Always non-nil.
  1292  var (
  1293  	testHookEnterRoundTrip   = nop
  1294  	testHookWaitResLoop      = nop
  1295  	testHookRoundTripRetried = nop
  1296  	testHookPrePendingDial   = nop
  1297  	testHookPostPendingDial  = nop
  1298  
  1299  	testHookMu                     sync.Locker = fakeLocker{} // guards following
  1300  	testHookReadLoopBeforeNextRead             = nop
  1301  )
  1302  
  1303  // beforeRespHeaderError is used to indicate when an IO error has occurred before
  1304  // any header data was received.
  1305  type beforeRespHeaderError struct {
  1306  	error
  1307  }
  1308  
  1309  func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
  1310  	testHookEnterRoundTrip()
  1311  	if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) {
  1312  		pc.t.putIdleConn(pc)
  1313  		return nil, errRequestCanceled
  1314  	}
  1315  	pc.lk.Lock()
  1316  	pc.numExpectedResponses++
  1317  	headerFn := pc.mutateHeaderFunc
  1318  	pc.lk.Unlock()
  1319  
  1320  	if headerFn != nil {
  1321  		headerFn(req.extraHeaders())
  1322  	}
  1323  
  1324  	// Ask for a compressed version if the caller didn't set their
  1325  	// own value for Accept-Encoding. We only attempt to
  1326  	// uncompress the gzip stream if we were the layer that
  1327  	// requested it.
  1328  	requestedGzip := false
  1329  	if !pc.t.DisableCompression &&
  1330  		req.Header.Get("Accept-Encoding") == "" &&
  1331  		req.Header.Get("Range") == "" &&
  1332  		req.Method != "HEAD" {
  1333  		// Request gzip only, not deflate. Deflate is ambiguous and
  1334  		// not as universally supported anyway.
  1335  		// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
  1336  		//
  1337  		// Note that we don't request this for HEAD requests,
  1338  		// due to a bug in nginx:
  1339  		//   http://trac.nginx.org/nginx/ticket/358
  1340  		//   https://golang.org/issue/5522
  1341  		//
  1342  		// We don't request gzip if the request is for a range, since
  1343  		// auto-decoding a portion of a gzipped document will just fail
  1344  		// anyway. See https://golang.org/issue/8923
  1345  		requestedGzip = true
  1346  		req.extraHeaders().Set("Accept-Encoding", "gzip")
  1347  	}
  1348  
  1349  	var continueCh chan struct{}
  1350  	if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
  1351  		continueCh = make(chan struct{}, 1)
  1352  	}
  1353  
  1354  	if pc.t.DisableKeepAlives {
  1355  		req.extraHeaders().Set("Connection", "close")
  1356  	}
  1357  
  1358  	// Write the request concurrently with waiting for a response,
  1359  	// in case the server decides to reply before reading our full
  1360  	// request body.
  1361  	writeErrCh := make(chan error, 1)
  1362  	pc.writech <- writeRequest{req, writeErrCh, continueCh}
  1363  
  1364  	resc := make(chan responseAndError, 1)
  1365  	pc.reqch <- requestAndChan{req.Request, resc, requestedGzip, continueCh}
  1366  
  1367  	var re responseAndError
  1368  	var respHeaderTimer <-chan time.Time
  1369  	cancelChan := req.Request.Cancel
  1370  WaitResponse:
  1371  	for {
  1372  		testHookWaitResLoop()
  1373  		select {
  1374  		case err := <-writeErrCh:
  1375  			if isNetWriteError(err) {
  1376  				// Issue 11745. If we failed to write the request
  1377  				// body, it's possible the server just heard enough
  1378  				// and already wrote to us. Prioritize the server's
  1379  				// response over returning a body write error.
  1380  				select {
  1381  				case re = <-resc:
  1382  					pc.close()
  1383  					break WaitResponse
  1384  				case <-time.After(50 * time.Millisecond):
  1385  					// Fall through.
  1386  				}
  1387  			}
  1388  			if err != nil {
  1389  				re = responseAndError{err: beforeRespHeaderError{err}}
  1390  				pc.close()
  1391  				break WaitResponse
  1392  			}
  1393  			if d := pc.t.ResponseHeaderTimeout; d > 0 {
  1394  				timer := time.NewTimer(d)
  1395  				defer timer.Stop() // prevent leaks
  1396  				respHeaderTimer = timer.C
  1397  			}
  1398  		case <-pc.closech:
  1399  			var err error
  1400  			if pc.isCanceled() {
  1401  				err = errRequestCanceled
  1402  			} else {
  1403  				err = beforeRespHeaderError{errClosed}
  1404  			}
  1405  			re = responseAndError{err: err}
  1406  			break WaitResponse
  1407  		case <-respHeaderTimer:
  1408  			pc.close()
  1409  			re = responseAndError{err: errTimeout}
  1410  			break WaitResponse
  1411  		case re = <-resc:
  1412  			break WaitResponse
  1413  		case <-cancelChan:
  1414  			pc.t.CancelRequest(req.Request)
  1415  			cancelChan = nil
  1416  		}
  1417  	}
  1418  
  1419  	if re.err != nil {
  1420  		pc.t.setReqCanceler(req.Request, nil)
  1421  	}
  1422  	return re.res(), re.err
  1423  }
  1424  
  1425  // markBroken marks a connection as broken (so it's not reused).
  1426  // It differs from close in that it doesn't close the underlying
  1427  // connection for use when it's still being read.
  1428  func (pc *persistConn) markBroken() {
  1429  	pc.lk.Lock()
  1430  	defer pc.lk.Unlock()
  1431  	pc.broken = true
  1432  }
  1433  
  1434  // markReused marks this connection as having been successfully used for a
  1435  // request and response.
  1436  func (pc *persistConn) markReused() {
  1437  	pc.lk.Lock()
  1438  	pc.reused = true
  1439  	pc.lk.Unlock()
  1440  }
  1441  
  1442  func (pc *persistConn) close() {
  1443  	pc.lk.Lock()
  1444  	defer pc.lk.Unlock()
  1445  	pc.closeLocked()
  1446  }
  1447  
  1448  func (pc *persistConn) closeLocked() {
  1449  	pc.broken = true
  1450  	if !pc.closed {
  1451  		pc.conn.Close()
  1452  		pc.closed = true
  1453  		close(pc.closech)
  1454  	}
  1455  	pc.mutateHeaderFunc = nil
  1456  }
  1457  
  1458  var portMap = map[string]string{
  1459  	"http":  "80",
  1460  	"https": "443",
  1461  }
  1462  
  1463  // canonicalAddr returns url.Host but always with a ":port" suffix
  1464  func canonicalAddr(url *url.URL) string {
  1465  	addr := url.Host
  1466  	if !hasPort(addr) {
  1467  		return addr + ":" + portMap[url.Scheme]
  1468  	}
  1469  	return addr
  1470  }
  1471  
  1472  // bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most
  1473  // once, right before its final (error-producing) Read or Close call
  1474  // returns. fn should return the new error to return from Read or Close.
  1475  //
  1476  // If earlyCloseFn is non-nil and Close is called before io.EOF is
  1477  // seen, earlyCloseFn is called instead of fn, and its return value is
  1478  // the return value from Close.
  1479  type bodyEOFSignal struct {
  1480  	body         io.ReadCloser
  1481  	mu           sync.Mutex        // guards following 4 fields
  1482  	closed       bool              // whether Close has been called
  1483  	rerr         error             // sticky Read error
  1484  	fn           func(error) error // err will be nil on Read io.EOF
  1485  	earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen
  1486  }
  1487  
  1488  func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
  1489  	es.mu.Lock()
  1490  	closed, rerr := es.closed, es.rerr
  1491  	es.mu.Unlock()
  1492  	if closed {
  1493  		return 0, errors.New("http: read on closed response body")
  1494  	}
  1495  	if rerr != nil {
  1496  		return 0, rerr
  1497  	}
  1498  
  1499  	n, err = es.body.Read(p)
  1500  	if err != nil {
  1501  		es.mu.Lock()
  1502  		defer es.mu.Unlock()
  1503  		if es.rerr == nil {
  1504  			es.rerr = err
  1505  		}
  1506  		err = es.condfn(err)
  1507  	}
  1508  	return
  1509  }
  1510  
  1511  func (es *bodyEOFSignal) Close() error {
  1512  	es.mu.Lock()
  1513  	defer es.mu.Unlock()
  1514  	if es.closed {
  1515  		return nil
  1516  	}
  1517  	es.closed = true
  1518  	if es.earlyCloseFn != nil && es.rerr != io.EOF {
  1519  		return es.earlyCloseFn()
  1520  	}
  1521  	err := es.body.Close()
  1522  	return es.condfn(err)
  1523  }
  1524  
  1525  // caller must hold es.mu.
  1526  func (es *bodyEOFSignal) condfn(err error) error {
  1527  	if es.fn == nil {
  1528  		return err
  1529  	}
  1530  	err = es.fn(err)
  1531  	es.fn = nil
  1532  	return err
  1533  }
  1534  
  1535  // gzipReader wraps a response body so it can lazily
  1536  // call gzip.NewReader on the first call to Read
  1537  type gzipReader struct {
  1538  	body io.ReadCloser // underlying Response.Body
  1539  	zr   io.Reader     // lazily-initialized gzip reader
  1540  }
  1541  
  1542  func (gz *gzipReader) Read(p []byte) (n int, err error) {
  1543  	if gz.zr == nil {
  1544  		gz.zr, err = gzip.NewReader(gz.body)
  1545  		if err != nil {
  1546  			return 0, err
  1547  		}
  1548  	}
  1549  	return gz.zr.Read(p)
  1550  }
  1551  
  1552  func (gz *gzipReader) Close() error {
  1553  	return gz.body.Close()
  1554  }
  1555  
  1556  type readerAndCloser struct {
  1557  	io.Reader
  1558  	io.Closer
  1559  }
  1560  
  1561  type tlsHandshakeTimeoutError struct{}
  1562  
  1563  func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
  1564  func (tlsHandshakeTimeoutError) Temporary() bool { return true }
  1565  func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
  1566  
  1567  type noteEOFReader struct {
  1568  	r      io.Reader
  1569  	sawEOF *bool
  1570  }
  1571  
  1572  func (nr noteEOFReader) Read(p []byte) (n int, err error) {
  1573  	n, err = nr.r.Read(p)
  1574  	if err == io.EOF {
  1575  		*nr.sawEOF = true
  1576  	}
  1577  	return
  1578  }
  1579  
  1580  // fakeLocker is a sync.Locker which does nothing. It's used to guard
  1581  // test-only fields when not under test, to avoid runtime atomic
  1582  // overhead.
  1583  type fakeLocker struct{}
  1584  
  1585  func (fakeLocker) Lock()   {}
  1586  func (fakeLocker) Unlock() {}
  1587  
  1588  func isNetWriteError(err error) bool {
  1589  	switch e := err.(type) {
  1590  	case *url.Error:
  1591  		return isNetWriteError(e.Err)
  1592  	case *net.OpError:
  1593  		return e.Op == "write"
  1594  	default:
  1595  		return false
  1596  	}
  1597  }
  1598  
  1599  // cloneTLSConfig returns a shallow clone of the exported
  1600  // fields of cfg, ignoring the unexported sync.Once, which
  1601  // contains a mutex and must not be copied.
  1602  //
  1603  // The cfg must not be in active use by tls.Server, or else
  1604  // there can still be a race with tls.Server updating SessionTicketKey
  1605  // and our copying it, and also a race with the server setting
  1606  // SessionTicketsDisabled=false on failure to set the random
  1607  // ticket key.
  1608  //
  1609  // If cfg is nil, a new zero tls.Config is returned.
  1610  func cloneTLSConfig(cfg *tls.Config) *tls.Config {
  1611  	if cfg == nil {
  1612  		return &tls.Config{}
  1613  	}
  1614  	return &tls.Config{
  1615  		Rand:                     cfg.Rand,
  1616  		Time:                     cfg.Time,
  1617  		Certificates:             cfg.Certificates,
  1618  		NameToCertificate:        cfg.NameToCertificate,
  1619  		GetCertificate:           cfg.GetCertificate,
  1620  		RootCAs:                  cfg.RootCAs,
  1621  		NextProtos:               cfg.NextProtos,
  1622  		ServerName:               cfg.ServerName,
  1623  		ClientAuth:               cfg.ClientAuth,
  1624  		ClientCAs:                cfg.ClientCAs,
  1625  		InsecureSkipVerify:       cfg.InsecureSkipVerify,
  1626  		CipherSuites:             cfg.CipherSuites,
  1627  		PreferServerCipherSuites: cfg.PreferServerCipherSuites,
  1628  		SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
  1629  		SessionTicketKey:         cfg.SessionTicketKey,
  1630  		ClientSessionCache:       cfg.ClientSessionCache,
  1631  		MinVersion:               cfg.MinVersion,
  1632  		MaxVersion:               cfg.MaxVersion,
  1633  		CurvePreferences:         cfg.CurvePreferences,
  1634  	}
  1635  }
  1636  
  1637  // cloneTLSClientConfig is like cloneTLSConfig but omits
  1638  // the fields SessionTicketsDisabled and SessionTicketKey.
  1639  // This makes it safe to call cloneTLSClientConfig on a config
  1640  // in active use by a server.
  1641  func cloneTLSClientConfig(cfg *tls.Config) *tls.Config {
  1642  	if cfg == nil {
  1643  		return &tls.Config{}
  1644  	}
  1645  	return &tls.Config{
  1646  		Rand:                     cfg.Rand,
  1647  		Time:                     cfg.Time,
  1648  		Certificates:             cfg.Certificates,
  1649  		NameToCertificate:        cfg.NameToCertificate,
  1650  		GetCertificate:           cfg.GetCertificate,
  1651  		RootCAs:                  cfg.RootCAs,
  1652  		NextProtos:               cfg.NextProtos,
  1653  		ServerName:               cfg.ServerName,
  1654  		ClientAuth:               cfg.ClientAuth,
  1655  		ClientCAs:                cfg.ClientCAs,
  1656  		InsecureSkipVerify:       cfg.InsecureSkipVerify,
  1657  		CipherSuites:             cfg.CipherSuites,
  1658  		PreferServerCipherSuites: cfg.PreferServerCipherSuites,
  1659  		ClientSessionCache:       cfg.ClientSessionCache,
  1660  		MinVersion:               cfg.MinVersion,
  1661  		MaxVersion:               cfg.MaxVersion,
  1662  		CurvePreferences:         cfg.CurvePreferences,
  1663  	}
  1664  }