github.com/sean-/go@v0.0.0-20151219100004-97f854cd7bb6/src/net/http/transport.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client implementation. See RFC 2616. 6 // 7 // This is the low-level Transport implementation of RoundTripper. 8 // The high-level interface is in client.go. 9 10 package http 11 12 import ( 13 "bufio" 14 "compress/gzip" 15 "crypto/tls" 16 "errors" 17 "fmt" 18 "io" 19 "log" 20 "net" 21 "net/url" 22 "os" 23 "strings" 24 "sync" 25 "time" 26 ) 27 28 // DefaultTransport is the default implementation of Transport and is 29 // used by DefaultClient. It establishes network connections as needed 30 // and caches them for reuse by subsequent calls. It uses HTTP proxies 31 // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and 32 // $no_proxy) environment variables. 33 var DefaultTransport RoundTripper = &Transport{ 34 Proxy: ProxyFromEnvironment, 35 Dial: (&net.Dialer{ 36 Timeout: 30 * time.Second, 37 KeepAlive: 30 * time.Second, 38 }).Dial, 39 TLSHandshakeTimeout: 10 * time.Second, 40 ExpectContinueTimeout: 1 * time.Second, 41 } 42 43 // DefaultMaxIdleConnsPerHost is the default value of Transport's 44 // MaxIdleConnsPerHost. 45 const DefaultMaxIdleConnsPerHost = 2 46 47 // Transport is an implementation of RoundTripper that supports HTTP, 48 // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 49 // Transport can also cache connections for future re-use. 50 type Transport struct { 51 idleMu sync.Mutex 52 wantIdle bool // user has requested to close all idle conns 53 idleConn map[connectMethodKey][]*persistConn 54 idleConnCh map[connectMethodKey]chan *persistConn 55 56 reqMu sync.Mutex 57 reqCanceler map[*Request]func() 58 59 altMu sync.RWMutex 60 altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper 61 62 // Proxy specifies a function to return a proxy for a given 63 // Request. If the function returns a non-nil error, the 64 // request is aborted with the provided error. 65 // If Proxy is nil or returns a nil *URL, no proxy is used. 66 Proxy func(*Request) (*url.URL, error) 67 68 // Dial specifies the dial function for creating unencrypted 69 // TCP connections. 70 // If Dial is nil, net.Dial is used. 71 Dial func(network, addr string) (net.Conn, error) 72 73 // DialTLS specifies an optional dial function for creating 74 // TLS connections for non-proxied HTTPS requests. 75 // 76 // If DialTLS is nil, Dial and TLSClientConfig are used. 77 // 78 // If DialTLS is set, the Dial hook is not used for HTTPS 79 // requests and the TLSClientConfig and TLSHandshakeTimeout 80 // are ignored. The returned net.Conn is assumed to already be 81 // past the TLS handshake. 82 DialTLS func(network, addr string) (net.Conn, error) 83 84 // TLSClientConfig specifies the TLS configuration to use with 85 // tls.Client. If nil, the default configuration is used. 86 TLSClientConfig *tls.Config 87 88 // TLSHandshakeTimeout specifies the maximum amount of time waiting to 89 // wait for a TLS handshake. Zero means no timeout. 90 TLSHandshakeTimeout time.Duration 91 92 // DisableKeepAlives, if true, prevents re-use of TCP connections 93 // between different HTTP requests. 94 DisableKeepAlives bool 95 96 // DisableCompression, if true, prevents the Transport from 97 // requesting compression with an "Accept-Encoding: gzip" 98 // request header when the Request contains no existing 99 // Accept-Encoding value. If the Transport requests gzip on 100 // its own and gets a gzipped response, it's transparently 101 // decoded in the Response.Body. However, if the user 102 // explicitly requested gzip it is not automatically 103 // uncompressed. 104 DisableCompression bool 105 106 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 107 // (keep-alive) to keep per-host. If zero, 108 // DefaultMaxIdleConnsPerHost is used. 109 MaxIdleConnsPerHost int 110 111 // ResponseHeaderTimeout, if non-zero, specifies the amount of 112 // time to wait for a server's response headers after fully 113 // writing the request (including its body, if any). This 114 // time does not include the time to read the response body. 115 ResponseHeaderTimeout time.Duration 116 117 // ExpectContinueTimeout, if non-zero, specifies the amount of 118 // time to wait for a server's first response headers after fully 119 // writing the request headers if the request has an 120 // "Expect: 100-continue" header. Zero means no timeout. 121 // This time does not include the time to send the request header. 122 ExpectContinueTimeout time.Duration 123 124 // TLSNextProto specifies how the Transport switches to an 125 // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN 126 // protocol negotiation. If Transport dials an TLS connection 127 // with a non-empty protocol name and TLSNextProto contains a 128 // map entry for that key (such as "h2"), then the func is 129 // called with the request's authority (such as "example.com" 130 // or "example.com:1234") and the TLS connection. The function 131 // must return a RoundTripper that then handles the request. 132 // If TLSNextProto is nil, HTTP/2 support is enabled automatically. 133 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 134 135 nextProtoOnce sync.Once // guards initialization of TLSNextProto (onceSetNextProtoDefaults) 136 137 // TODO: tunable on global max cached connections 138 // TODO: tunable on timeout on cached connections 139 } 140 141 // onceSetNextProtoDefaults initializes TLSNextProto. 142 // It must be called via t.nextProtoOnce.Do. 143 func (t *Transport) onceSetNextProtoDefaults() { 144 if strings.Contains(os.Getenv("GODEBUG"), "h2client=0") { 145 return 146 } 147 if t.TLSNextProto != nil { 148 return 149 } 150 err := http2ConfigureTransport(t) 151 if err != nil { 152 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 153 } 154 } 155 156 // ProxyFromEnvironment returns the URL of the proxy to use for a 157 // given request, as indicated by the environment variables 158 // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 159 // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https 160 // requests. 161 // 162 // The environment values may be either a complete URL or a 163 // "host[:port]", in which case the "http" scheme is assumed. 164 // An error is returned if the value is a different form. 165 // 166 // A nil URL and nil error are returned if no proxy is defined in the 167 // environment, or a proxy should not be used for the given request, 168 // as defined by NO_PROXY. 169 // 170 // As a special case, if req.URL.Host is "localhost" (with or without 171 // a port number), then a nil URL and nil error will be returned. 172 func ProxyFromEnvironment(req *Request) (*url.URL, error) { 173 var proxy string 174 if req.URL.Scheme == "https" { 175 proxy = httpsProxyEnv.Get() 176 } 177 if proxy == "" { 178 proxy = httpProxyEnv.Get() 179 } 180 if proxy == "" { 181 return nil, nil 182 } 183 if !useProxy(canonicalAddr(req.URL)) { 184 return nil, nil 185 } 186 proxyURL, err := url.Parse(proxy) 187 if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { 188 // proxy was bogus. Try prepending "http://" to it and 189 // see if that parses correctly. If not, we fall 190 // through and complain about the original one. 191 if proxyURL, err := url.Parse("http://" + proxy); err == nil { 192 return proxyURL, nil 193 } 194 } 195 if err != nil { 196 return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) 197 } 198 return proxyURL, nil 199 } 200 201 // ProxyURL returns a proxy function (for use in a Transport) 202 // that always returns the same URL. 203 func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 204 return func(*Request) (*url.URL, error) { 205 return fixedURL, nil 206 } 207 } 208 209 // transportRequest is a wrapper around a *Request that adds 210 // optional extra headers to write. 211 type transportRequest struct { 212 *Request // original request, not to be mutated 213 extra Header // extra headers to write, or nil 214 } 215 216 func (tr *transportRequest) extraHeaders() Header { 217 if tr.extra == nil { 218 tr.extra = make(Header) 219 } 220 return tr.extra 221 } 222 223 // RoundTrip implements the RoundTripper interface. 224 // 225 // For higher-level HTTP client support (such as handling of cookies 226 // and redirects), see Get, Post, and the Client type. 227 func (t *Transport) RoundTrip(req *Request) (*Response, error) { 228 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 229 if req.URL == nil { 230 req.closeBody() 231 return nil, errors.New("http: nil Request.URL") 232 } 233 if req.Header == nil { 234 req.closeBody() 235 return nil, errors.New("http: nil Request.Header") 236 } 237 // TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex 238 t.altMu.RLock() 239 altRT := t.altProto[req.URL.Scheme] 240 t.altMu.RUnlock() 241 if altRT != nil { 242 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 243 return resp, err 244 } 245 } 246 if s := req.URL.Scheme; s != "http" && s != "https" { 247 req.closeBody() 248 return nil, &badStringError{"unsupported protocol scheme", s} 249 } 250 if req.Method != "" && !validMethod(req.Method) { 251 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 252 } 253 if req.URL.Host == "" { 254 req.closeBody() 255 return nil, errors.New("http: no Host in request URL") 256 } 257 258 for { 259 // treq gets modified by roundTrip, so we need to recreate for each retry. 260 treq := &transportRequest{Request: req} 261 cm, err := t.connectMethodForRequest(treq) 262 if err != nil { 263 req.closeBody() 264 return nil, err 265 } 266 267 // Get the cached or newly-created connection to either the 268 // host (for http or https), the http proxy, or the http proxy 269 // pre-CONNECTed to https server. In any case, we'll be ready 270 // to send it requests. 271 pconn, err := t.getConn(req, cm) 272 if err != nil { 273 t.setReqCanceler(req, nil) 274 req.closeBody() 275 return nil, err 276 } 277 278 var resp *Response 279 if pconn.alt != nil { 280 // HTTP/2 path. 281 resp, err = pconn.alt.RoundTrip(req) 282 } else { 283 resp, err = pconn.roundTrip(treq) 284 } 285 if err == nil { 286 return resp, nil 287 } 288 if err := checkTransportResend(err, req, pconn); err != nil { 289 return nil, err 290 } 291 testHookRoundTripRetried() 292 } 293 } 294 295 // checkTransportResend checks whether a failed HTTP request can be 296 // resent on a new connection. The non-nil input error is the error from 297 // roundTrip, which might be wrapped in a beforeRespHeaderError error. 298 // 299 // The return value is err or the unwrapped error inside a 300 // beforeRespHeaderError. 301 func checkTransportResend(err error, req *Request, pconn *persistConn) error { 302 brhErr, ok := err.(beforeRespHeaderError) 303 if !ok { 304 return err 305 } 306 err = brhErr.error // unwrap the custom error in case we return it 307 if err != errMissingHost && pconn.isReused() && req.isReplayable() { 308 // If we try to reuse a connection that the server is in the process of 309 // closing, we may end up successfully writing out our request (or a 310 // portion of our request) only to find a connection error when we try to 311 // read from (or finish writing to) the socket. 312 313 // There can be a race between the socket pool checking whether a socket 314 // is still connected, receiving the FIN, and sending/reading data on a 315 // reused socket. If we receive the FIN between the connectedness check 316 // and writing/reading from the socket, we may first learn the socket is 317 // disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most 318 // likely happen when trying to retrieve its IP address. See 319 // http://crbug.com/105824 for more details. 320 321 // We resend a request only if we reused a keep-alive connection and did 322 // not yet receive any header data. This automatically prevents an 323 // infinite resend loop because we'll run out of the cached keep-alive 324 // connections eventually. 325 return nil 326 } 327 return err 328 } 329 330 // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 331 var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 332 333 // RegisterProtocol registers a new protocol with scheme. 334 // The Transport will pass requests using the given scheme to rt. 335 // It is rt's responsibility to simulate HTTP request semantics. 336 // 337 // RegisterProtocol can be used by other packages to provide 338 // implementations of protocol schemes like "ftp" or "file". 339 // 340 // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will 341 // handle the RoundTrip itself for that one request, as if the 342 // protocol were not registered. 343 func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 344 t.altMu.Lock() 345 defer t.altMu.Unlock() 346 if t.altProto == nil { 347 t.altProto = make(map[string]RoundTripper) 348 } 349 if _, exists := t.altProto[scheme]; exists { 350 panic("protocol " + scheme + " already registered") 351 } 352 t.altProto[scheme] = rt 353 } 354 355 // CloseIdleConnections closes any connections which were previously 356 // connected from previous requests but are now sitting idle in 357 // a "keep-alive" state. It does not interrupt any connections currently 358 // in use. 359 func (t *Transport) CloseIdleConnections() { 360 t.idleMu.Lock() 361 m := t.idleConn 362 t.idleConn = nil 363 t.idleConnCh = nil 364 t.wantIdle = true 365 t.idleMu.Unlock() 366 for _, conns := range m { 367 for _, pconn := range conns { 368 pconn.close() 369 } 370 } 371 } 372 373 // CancelRequest cancels an in-flight request by closing its connection. 374 // CancelRequest should only be called after RoundTrip has returned. 375 func (t *Transport) CancelRequest(req *Request) { 376 t.reqMu.Lock() 377 cancel := t.reqCanceler[req] 378 delete(t.reqCanceler, req) 379 t.reqMu.Unlock() 380 if cancel != nil { 381 cancel() 382 } 383 } 384 385 // 386 // Private implementation past this point. 387 // 388 389 var ( 390 httpProxyEnv = &envOnce{ 391 names: []string{"HTTP_PROXY", "http_proxy"}, 392 } 393 httpsProxyEnv = &envOnce{ 394 names: []string{"HTTPS_PROXY", "https_proxy"}, 395 } 396 noProxyEnv = &envOnce{ 397 names: []string{"NO_PROXY", "no_proxy"}, 398 } 399 ) 400 401 // envOnce looks up an environment variable (optionally by multiple 402 // names) once. It mitigates expensive lookups on some platforms 403 // (e.g. Windows). 404 type envOnce struct { 405 names []string 406 once sync.Once 407 val string 408 } 409 410 func (e *envOnce) Get() string { 411 e.once.Do(e.init) 412 return e.val 413 } 414 415 func (e *envOnce) init() { 416 for _, n := range e.names { 417 e.val = os.Getenv(n) 418 if e.val != "" { 419 return 420 } 421 } 422 } 423 424 // reset is used by tests 425 func (e *envOnce) reset() { 426 e.once = sync.Once{} 427 e.val = "" 428 } 429 430 func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 431 cm.targetScheme = treq.URL.Scheme 432 cm.targetAddr = canonicalAddr(treq.URL) 433 if t.Proxy != nil { 434 cm.proxyURL, err = t.Proxy(treq.Request) 435 } 436 return cm, err 437 } 438 439 // proxyAuth returns the Proxy-Authorization header to set 440 // on requests, if applicable. 441 func (cm *connectMethod) proxyAuth() string { 442 if cm.proxyURL == nil { 443 return "" 444 } 445 if u := cm.proxyURL.User; u != nil { 446 username := u.Username() 447 password, _ := u.Password() 448 return "Basic " + basicAuth(username, password) 449 } 450 return "" 451 } 452 453 // putIdleConn adds pconn to the list of idle persistent connections awaiting 454 // a new request. 455 // If pconn is no longer needed or not in a good state, putIdleConn 456 // returns false. 457 func (t *Transport) putIdleConn(pconn *persistConn) bool { 458 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 459 pconn.close() 460 return false 461 } 462 if pconn.isBroken() { 463 return false 464 } 465 key := pconn.cacheKey 466 max := t.MaxIdleConnsPerHost 467 if max == 0 { 468 max = DefaultMaxIdleConnsPerHost 469 } 470 pconn.markReused() 471 t.idleMu.Lock() 472 473 waitingDialer := t.idleConnCh[key] 474 select { 475 case waitingDialer <- pconn: 476 // We're done with this pconn and somebody else is 477 // currently waiting for a conn of this type (they're 478 // actively dialing, but this conn is ready 479 // first). Chrome calls this socket late binding. See 480 // https://insouciant.org/tech/connection-management-in-chromium/ 481 t.idleMu.Unlock() 482 return true 483 default: 484 if waitingDialer != nil { 485 // They had populated this, but their dial won 486 // first, so we can clean up this map entry. 487 delete(t.idleConnCh, key) 488 } 489 } 490 if t.wantIdle { 491 t.idleMu.Unlock() 492 pconn.close() 493 return false 494 } 495 if t.idleConn == nil { 496 t.idleConn = make(map[connectMethodKey][]*persistConn) 497 } 498 if len(t.idleConn[key]) >= max { 499 t.idleMu.Unlock() 500 pconn.close() 501 return false 502 } 503 for _, exist := range t.idleConn[key] { 504 if exist == pconn { 505 log.Fatalf("dup idle pconn %p in freelist", pconn) 506 } 507 } 508 t.idleConn[key] = append(t.idleConn[key], pconn) 509 t.idleMu.Unlock() 510 return true 511 } 512 513 // getIdleConnCh returns a channel to receive and return idle 514 // persistent connection for the given connectMethod. 515 // It may return nil, if persistent connections are not being used. 516 func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { 517 if t.DisableKeepAlives { 518 return nil 519 } 520 key := cm.key() 521 t.idleMu.Lock() 522 defer t.idleMu.Unlock() 523 t.wantIdle = false 524 if t.idleConnCh == nil { 525 t.idleConnCh = make(map[connectMethodKey]chan *persistConn) 526 } 527 ch, ok := t.idleConnCh[key] 528 if !ok { 529 ch = make(chan *persistConn) 530 t.idleConnCh[key] = ch 531 } 532 return ch 533 } 534 535 func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) { 536 key := cm.key() 537 t.idleMu.Lock() 538 defer t.idleMu.Unlock() 539 if t.idleConn == nil { 540 return nil 541 } 542 for { 543 pconns, ok := t.idleConn[key] 544 if !ok { 545 return nil 546 } 547 if len(pconns) == 1 { 548 pconn = pconns[0] 549 delete(t.idleConn, key) 550 } else { 551 // 2 or more cached connections; pop last 552 // TODO: queue? 553 pconn = pconns[len(pconns)-1] 554 t.idleConn[key] = pconns[:len(pconns)-1] 555 } 556 if !pconn.isBroken() { 557 return 558 } 559 } 560 } 561 562 func (t *Transport) setReqCanceler(r *Request, fn func()) { 563 t.reqMu.Lock() 564 defer t.reqMu.Unlock() 565 if t.reqCanceler == nil { 566 t.reqCanceler = make(map[*Request]func()) 567 } 568 if fn != nil { 569 t.reqCanceler[r] = fn 570 } else { 571 delete(t.reqCanceler, r) 572 } 573 } 574 575 // replaceReqCanceler replaces an existing cancel function. If there is no cancel function 576 // for the request, we don't set the function and return false. 577 // Since CancelRequest will clear the canceler, we can use the return value to detect if 578 // the request was canceled since the last setReqCancel call. 579 func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool { 580 t.reqMu.Lock() 581 defer t.reqMu.Unlock() 582 _, ok := t.reqCanceler[r] 583 if !ok { 584 return false 585 } 586 if fn != nil { 587 t.reqCanceler[r] = fn 588 } else { 589 delete(t.reqCanceler, r) 590 } 591 return true 592 } 593 594 func (t *Transport) dial(network, addr string) (c net.Conn, err error) { 595 if t.Dial != nil { 596 return t.Dial(network, addr) 597 } 598 return net.Dial(network, addr) 599 } 600 601 // getConn dials and creates a new persistConn to the target as 602 // specified in the connectMethod. This includes doing a proxy CONNECT 603 // and/or setting up TLS. If this doesn't return an error, the persistConn 604 // is ready to write requests to. 605 func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) { 606 if pc := t.getIdleConn(cm); pc != nil { 607 // set request canceler to some non-nil function so we 608 // can detect whether it was cleared between now and when 609 // we enter roundTrip 610 t.setReqCanceler(req, func() {}) 611 return pc, nil 612 } 613 614 type dialRes struct { 615 pc *persistConn 616 err error 617 } 618 dialc := make(chan dialRes) 619 620 // Copy these hooks so we don't race on the postPendingDial in 621 // the goroutine we launch. Issue 11136. 622 testHookPrePendingDial := testHookPrePendingDial 623 testHookPostPendingDial := testHookPostPendingDial 624 625 handlePendingDial := func() { 626 testHookPrePendingDial() 627 go func() { 628 if v := <-dialc; v.err == nil { 629 t.putIdleConn(v.pc) 630 } 631 testHookPostPendingDial() 632 }() 633 } 634 635 cancelc := make(chan struct{}) 636 t.setReqCanceler(req, func() { close(cancelc) }) 637 638 go func() { 639 pc, err := t.dialConn(cm) 640 dialc <- dialRes{pc, err} 641 }() 642 643 idleConnCh := t.getIdleConnCh(cm) 644 select { 645 case v := <-dialc: 646 // Our dial finished. 647 return v.pc, v.err 648 case pc := <-idleConnCh: 649 // Another request finished first and its net.Conn 650 // became available before our dial. Or somebody 651 // else's dial that they didn't use. 652 // But our dial is still going, so give it away 653 // when it finishes: 654 handlePendingDial() 655 return pc, nil 656 case <-req.Cancel: 657 handlePendingDial() 658 return nil, errors.New("net/http: request canceled while waiting for connection") 659 case <-cancelc: 660 handlePendingDial() 661 return nil, errors.New("net/http: request canceled while waiting for connection") 662 } 663 } 664 665 func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { 666 pconn := &persistConn{ 667 t: t, 668 cacheKey: cm.key(), 669 reqch: make(chan requestAndChan, 1), 670 writech: make(chan writeRequest, 1), 671 closech: make(chan struct{}), 672 writeErrCh: make(chan error, 1), 673 } 674 tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil 675 if tlsDial { 676 var err error 677 pconn.conn, err = t.DialTLS("tcp", cm.addr()) 678 if err != nil { 679 return nil, err 680 } 681 if tc, ok := pconn.conn.(*tls.Conn); ok { 682 cs := tc.ConnectionState() 683 pconn.tlsState = &cs 684 } 685 } else { 686 conn, err := t.dial("tcp", cm.addr()) 687 if err != nil { 688 if cm.proxyURL != nil { 689 err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) 690 } 691 return nil, err 692 } 693 pconn.conn = conn 694 } 695 696 // Proxy setup. 697 switch { 698 case cm.proxyURL == nil: 699 // Do nothing. Not using a proxy. 700 case cm.targetScheme == "http": 701 pconn.isProxy = true 702 if pa := cm.proxyAuth(); pa != "" { 703 pconn.mutateHeaderFunc = func(h Header) { 704 h.Set("Proxy-Authorization", pa) 705 } 706 } 707 case cm.targetScheme == "https": 708 conn := pconn.conn 709 connectReq := &Request{ 710 Method: "CONNECT", 711 URL: &url.URL{Opaque: cm.targetAddr}, 712 Host: cm.targetAddr, 713 Header: make(Header), 714 } 715 if pa := cm.proxyAuth(); pa != "" { 716 connectReq.Header.Set("Proxy-Authorization", pa) 717 } 718 connectReq.Write(conn) 719 720 // Read response. 721 // Okay to use and discard buffered reader here, because 722 // TLS server will not speak until spoken to. 723 br := bufio.NewReader(conn) 724 resp, err := ReadResponse(br, connectReq) 725 if err != nil { 726 conn.Close() 727 return nil, err 728 } 729 if resp.StatusCode != 200 { 730 f := strings.SplitN(resp.Status, " ", 2) 731 conn.Close() 732 return nil, errors.New(f[1]) 733 } 734 } 735 736 if cm.targetScheme == "https" && !tlsDial { 737 // Initiate TLS and check remote host name against certificate. 738 cfg := cloneTLSClientConfig(t.TLSClientConfig) 739 if cfg.ServerName == "" { 740 cfg.ServerName = cm.tlsHost() 741 } 742 plainConn := pconn.conn 743 tlsConn := tls.Client(plainConn, cfg) 744 errc := make(chan error, 2) 745 var timer *time.Timer // for canceling TLS handshake 746 if d := t.TLSHandshakeTimeout; d != 0 { 747 timer = time.AfterFunc(d, func() { 748 errc <- tlsHandshakeTimeoutError{} 749 }) 750 } 751 go func() { 752 err := tlsConn.Handshake() 753 if timer != nil { 754 timer.Stop() 755 } 756 errc <- err 757 }() 758 if err := <-errc; err != nil { 759 plainConn.Close() 760 return nil, err 761 } 762 if !cfg.InsecureSkipVerify { 763 if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { 764 plainConn.Close() 765 return nil, err 766 } 767 } 768 cs := tlsConn.ConnectionState() 769 pconn.tlsState = &cs 770 pconn.conn = tlsConn 771 } 772 773 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 774 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 775 return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil 776 } 777 } 778 779 pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF}) 780 pconn.bw = bufio.NewWriter(pconn.conn) 781 go pconn.readLoop() 782 go pconn.writeLoop() 783 return pconn, nil 784 } 785 786 // useProxy reports whether requests to addr should use a proxy, 787 // according to the NO_PROXY or no_proxy environment variable. 788 // addr is always a canonicalAddr with a host and port. 789 func useProxy(addr string) bool { 790 if len(addr) == 0 { 791 return true 792 } 793 host, _, err := net.SplitHostPort(addr) 794 if err != nil { 795 return false 796 } 797 if host == "localhost" { 798 return false 799 } 800 if ip := net.ParseIP(host); ip != nil { 801 if ip.IsLoopback() { 802 return false 803 } 804 } 805 806 no_proxy := noProxyEnv.Get() 807 if no_proxy == "*" { 808 return false 809 } 810 811 addr = strings.ToLower(strings.TrimSpace(addr)) 812 if hasPort(addr) { 813 addr = addr[:strings.LastIndex(addr, ":")] 814 } 815 816 for _, p := range strings.Split(no_proxy, ",") { 817 p = strings.ToLower(strings.TrimSpace(p)) 818 if len(p) == 0 { 819 continue 820 } 821 if hasPort(p) { 822 p = p[:strings.LastIndex(p, ":")] 823 } 824 if addr == p { 825 return false 826 } 827 if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { 828 // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" 829 return false 830 } 831 if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { 832 // no_proxy "foo.com" matches "bar.foo.com" 833 return false 834 } 835 } 836 return true 837 } 838 839 // connectMethod is the map key (in its String form) for keeping persistent 840 // TCP connections alive for subsequent HTTP requests. 841 // 842 // A connect method may be of the following types: 843 // 844 // Cache key form Description 845 // ----------------- ------------------------- 846 // |http|foo.com http directly to server, no proxy 847 // |https|foo.com https directly to server, no proxy 848 // http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 849 // http://proxy.com|http http to proxy, http to anywhere after that 850 // 851 // Note: no support to https to the proxy yet. 852 // 853 type connectMethod struct { 854 proxyURL *url.URL // nil for no proxy, else full proxy URL 855 targetScheme string // "http" or "https" 856 targetAddr string // Not used if proxy + http targetScheme (4th example in table) 857 } 858 859 func (cm *connectMethod) key() connectMethodKey { 860 proxyStr := "" 861 targetAddr := cm.targetAddr 862 if cm.proxyURL != nil { 863 proxyStr = cm.proxyURL.String() 864 if cm.targetScheme == "http" { 865 targetAddr = "" 866 } 867 } 868 return connectMethodKey{ 869 proxy: proxyStr, 870 scheme: cm.targetScheme, 871 addr: targetAddr, 872 } 873 } 874 875 // addr returns the first hop "host:port" to which we need to TCP connect. 876 func (cm *connectMethod) addr() string { 877 if cm.proxyURL != nil { 878 return canonicalAddr(cm.proxyURL) 879 } 880 return cm.targetAddr 881 } 882 883 // tlsHost returns the host name to match against the peer's 884 // TLS certificate. 885 func (cm *connectMethod) tlsHost() string { 886 h := cm.targetAddr 887 if hasPort(h) { 888 h = h[:strings.LastIndex(h, ":")] 889 } 890 return h 891 } 892 893 // connectMethodKey is the map key version of connectMethod, with a 894 // stringified proxy URL (or the empty string) instead of a pointer to 895 // a URL. 896 type connectMethodKey struct { 897 proxy, scheme, addr string 898 } 899 900 func (k connectMethodKey) String() string { 901 // Only used by tests. 902 return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr) 903 } 904 905 // persistConn wraps a connection, usually a persistent one 906 // (but may be used for non-keep-alive requests as well) 907 type persistConn struct { 908 // alt optionally specifies the TLS NextProto RoundTripper. 909 // This is used for HTTP/2 today and future protocol laters. 910 // If it's non-nil, the rest of the fields are unused. 911 alt RoundTripper 912 913 t *Transport 914 cacheKey connectMethodKey 915 conn net.Conn 916 tlsState *tls.ConnectionState 917 br *bufio.Reader // from conn 918 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 919 bw *bufio.Writer // to conn 920 reqch chan requestAndChan // written by roundTrip; read by readLoop 921 writech chan writeRequest // written by roundTrip; read by writeLoop 922 closech chan struct{} // closed when conn closed 923 isProxy bool 924 // writeErrCh passes the request write error (usually nil) 925 // from the writeLoop goroutine to the readLoop which passes 926 // it off to the res.Body reader, which then uses it to decide 927 // whether or not a connection can be reused. Issue 7569. 928 writeErrCh chan error 929 930 lk sync.Mutex // guards following fields 931 numExpectedResponses int 932 closed bool // whether conn has been closed 933 broken bool // an error has happened on this connection; marked broken so it's not reused. 934 canceled bool // whether this conn was broken due a CancelRequest 935 reused bool // whether conn has had successful request/response and is being reused. 936 // mutateHeaderFunc is an optional func to modify extra 937 // headers on each outbound request before it's written. (the 938 // original Request given to RoundTrip is not modified) 939 mutateHeaderFunc func(Header) 940 } 941 942 // isBroken reports whether this connection is in a known broken state. 943 func (pc *persistConn) isBroken() bool { 944 pc.lk.Lock() 945 b := pc.broken 946 pc.lk.Unlock() 947 return b 948 } 949 950 // isCanceled reports whether this connection was closed due to CancelRequest. 951 func (pc *persistConn) isCanceled() bool { 952 pc.lk.Lock() 953 defer pc.lk.Unlock() 954 return pc.canceled 955 } 956 957 // isReused reports whether this connection is in a known broken state. 958 func (pc *persistConn) isReused() bool { 959 pc.lk.Lock() 960 r := pc.reused 961 pc.lk.Unlock() 962 return r 963 } 964 965 func (pc *persistConn) cancelRequest() { 966 pc.lk.Lock() 967 defer pc.lk.Unlock() 968 pc.canceled = true 969 pc.closeLocked() 970 } 971 972 func (pc *persistConn) readLoop() { 973 defer pc.close() 974 975 // eofc is used to block caller goroutines reading from Response.Body 976 // at EOF until this goroutines has (potentially) added the connection 977 // back to the idle pool. 978 eofc := make(chan struct{}) 979 defer close(eofc) // unblock reader on errors 980 981 // Read this once, before loop starts. (to avoid races in tests) 982 testHookMu.Lock() 983 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 984 testHookMu.Unlock() 985 986 alive := true 987 for alive { 988 _, err := pc.br.Peek(1) 989 if err != nil { 990 err = beforeRespHeaderError{err} 991 } 992 993 pc.lk.Lock() 994 if pc.numExpectedResponses == 0 { 995 pc.readLoopPeekFailLocked(err) 996 pc.lk.Unlock() 997 return 998 } 999 pc.lk.Unlock() 1000 1001 rc := <-pc.reqch 1002 1003 var resp *Response 1004 if err == nil { 1005 resp, err = pc.readResponse(rc) 1006 } 1007 1008 if err != nil { 1009 // If we won't be able to retry this request later (from the 1010 // roundTrip goroutine), mark it as done now. 1011 // BEFORE the send on rc.ch, as the client might re-use the 1012 // same *Request pointer, and we don't want to set call 1013 // t.setReqCanceler from this persistConn while the Transport 1014 // potentially spins up a different persistConn for the 1015 // caller's subsequent request. 1016 if checkTransportResend(err, rc.req, pc) != nil { 1017 pc.t.setReqCanceler(rc.req, nil) 1018 } 1019 rc.ch <- responseAndError{err: err} 1020 return 1021 } 1022 1023 pc.lk.Lock() 1024 pc.numExpectedResponses-- 1025 pc.lk.Unlock() 1026 1027 hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 1028 1029 if resp.Close || rc.req.Close || resp.StatusCode <= 199 { 1030 // Don't do keep-alive on error if either party requested a close 1031 // or we get an unexpected informational (1xx) response. 1032 // StatusCode 100 is already handled above. 1033 alive = false 1034 } 1035 1036 if !hasBody { 1037 pc.t.setReqCanceler(rc.req, nil) 1038 resc := make(chan *Response) // unbuffered matters; see below 1039 rc.ch <- responseAndError{ch: resc} // buffered send 1040 1041 // Put the idle conn back into the pool before we send the response 1042 // so if they process it quickly and make another request, they'll 1043 // get this same conn. But we use the unbuffered channel 'rc' 1044 // to guarantee that persistConn.roundTrip got out of its select 1045 // potentially waiting for this persistConn to close. 1046 // but after 1047 alive = alive && 1048 !pc.sawEOF && 1049 pc.wroteRequest() && 1050 pc.t.putIdleConn(pc) 1051 1052 resc <- resp // unbuffered send 1053 1054 // Now that they've read from the unbuffered channel, they're safely 1055 // out of the select that also waits on this goroutine to die, so 1056 // we're allowed to exit now if needed (if alive is false) 1057 testHookReadLoopBeforeNextRead() 1058 continue 1059 } 1060 1061 if rc.addedGzip { 1062 maybeUngzipResponse(resp) 1063 } 1064 resp.Body = &bodyEOFSignal{body: resp.Body} 1065 1066 waitForBodyRead := make(chan bool, 2) 1067 resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error { 1068 waitForBodyRead <- false 1069 return nil 1070 } 1071 resp.Body.(*bodyEOFSignal).fn = func(err error) error { 1072 isEOF := err == io.EOF 1073 waitForBodyRead <- isEOF 1074 if isEOF { 1075 <-eofc // see comment above eofc declaration 1076 } else if err != nil && pc.isCanceled() { 1077 return errRequestCanceled 1078 } 1079 return err 1080 } 1081 1082 rc.ch <- responseAndError{r: resp} 1083 1084 // Before looping back to the top of this function and peeking on 1085 // the bufio.Reader, wait for the caller goroutine to finish 1086 // reading the response body. (or for cancelation or death) 1087 select { 1088 case bodyEOF := <-waitForBodyRead: 1089 pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool 1090 alive = alive && 1091 bodyEOF && 1092 !pc.sawEOF && 1093 pc.wroteRequest() && 1094 pc.t.putIdleConn(pc) 1095 if bodyEOF { 1096 eofc <- struct{}{} 1097 } 1098 case <-rc.req.Cancel: 1099 alive = false 1100 pc.t.CancelRequest(rc.req) 1101 case <-pc.closech: 1102 alive = false 1103 } 1104 1105 testHookReadLoopBeforeNextRead() 1106 } 1107 } 1108 1109 func maybeUngzipResponse(resp *Response) { 1110 if resp.Header.Get("Content-Encoding") == "gzip" { 1111 resp.Header.Del("Content-Encoding") 1112 resp.Header.Del("Content-Length") 1113 resp.ContentLength = -1 1114 resp.Body = &gzipReader{body: resp.Body} 1115 } 1116 } 1117 1118 func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 1119 if pc.closed { 1120 return 1121 } 1122 if n := pc.br.Buffered(); n > 0 { 1123 buf, _ := pc.br.Peek(n) 1124 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 1125 } 1126 pc.closeLocked() 1127 } 1128 1129 // readResponse reads an HTTP response (or two, in the case of "Expect: 1130 // 100-continue") from the server. It returns the final non-100 one. 1131 func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) { 1132 resp, err = ReadResponse(pc.br, rc.req) 1133 if err != nil { 1134 return 1135 } 1136 if rc.continueCh != nil { 1137 if resp.StatusCode == 100 { 1138 rc.continueCh <- struct{}{} 1139 } else { 1140 close(rc.continueCh) 1141 } 1142 } 1143 if resp.StatusCode == 100 { 1144 resp, err = ReadResponse(pc.br, rc.req) 1145 if err != nil { 1146 return 1147 } 1148 } 1149 resp.TLS = pc.tlsState 1150 return 1151 } 1152 1153 // waitForContinue returns the function to block until 1154 // any response, timeout or connection close. After any of them, 1155 // the function returns a bool which indicates if the body should be sent. 1156 func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 1157 if continueCh == nil { 1158 return nil 1159 } 1160 return func() bool { 1161 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 1162 defer timer.Stop() 1163 1164 select { 1165 case _, ok := <-continueCh: 1166 return ok 1167 case <-timer.C: 1168 return true 1169 case <-pc.closech: 1170 return false 1171 } 1172 } 1173 } 1174 1175 func (pc *persistConn) writeLoop() { 1176 for { 1177 select { 1178 case wr := <-pc.writech: 1179 if pc.isBroken() { 1180 wr.ch <- errors.New("http: can't write HTTP request on broken connection") 1181 continue 1182 } 1183 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 1184 if err == nil { 1185 err = pc.bw.Flush() 1186 } 1187 if err != nil { 1188 pc.markBroken() 1189 wr.req.Request.closeBody() 1190 } 1191 pc.writeErrCh <- err // to the body reader, which might recycle us 1192 wr.ch <- err // to the roundTrip function 1193 case <-pc.closech: 1194 return 1195 } 1196 } 1197 } 1198 1199 // wroteRequest is a check before recycling a connection that the previous write 1200 // (from writeLoop above) happened and was successful. 1201 func (pc *persistConn) wroteRequest() bool { 1202 select { 1203 case err := <-pc.writeErrCh: 1204 // Common case: the write happened well before the response, so 1205 // avoid creating a timer. 1206 return err == nil 1207 default: 1208 // Rare case: the request was written in writeLoop above but 1209 // before it could send to pc.writeErrCh, the reader read it 1210 // all, processed it, and called us here. In this case, give the 1211 // write goroutine a bit of time to finish its send. 1212 // 1213 // Less rare case: We also get here in the legitimate case of 1214 // Issue 7569, where the writer is still writing (or stalled), 1215 // but the server has already replied. In this case, we don't 1216 // want to wait too long, and we want to return false so this 1217 // connection isn't re-used. 1218 select { 1219 case err := <-pc.writeErrCh: 1220 return err == nil 1221 case <-time.After(50 * time.Millisecond): 1222 return false 1223 } 1224 } 1225 } 1226 1227 // responseAndError is how the goroutine reading from an HTTP/1 server 1228 // communicates with the goroutine doing the RoundTrip. 1229 type responseAndError struct { 1230 ch chan *Response // if non-nil, res should be read from here 1231 r *Response // else use this response (see res method) 1232 err error 1233 } 1234 1235 func (re responseAndError) res() *Response { 1236 switch { 1237 case re.err != nil: 1238 return nil 1239 case re.ch != nil: 1240 return <-re.ch 1241 default: 1242 return re.r 1243 } 1244 } 1245 1246 type requestAndChan struct { 1247 req *Request 1248 ch chan responseAndError 1249 1250 // did the Transport (as opposed to the client code) add an 1251 // Accept-Encoding gzip header? only if it we set it do 1252 // we transparently decode the gzip. 1253 addedGzip bool 1254 1255 // Optional blocking chan for Expect: 100-continue (for send). 1256 // If the request has an "Expect: 100-continue" header and 1257 // the server responds 100 Continue, readLoop send a value 1258 // to writeLoop via this chan. 1259 continueCh chan<- struct{} 1260 } 1261 1262 // A writeRequest is sent by the readLoop's goroutine to the 1263 // writeLoop's goroutine to write a request while the read loop 1264 // concurrently waits on both the write response and the server's 1265 // reply. 1266 type writeRequest struct { 1267 req *transportRequest 1268 ch chan<- error 1269 1270 // Optional blocking chan for Expect: 100-continue (for recieve). 1271 // If not nil, writeLoop blocks sending request body until 1272 // it receives from this chan. 1273 continueCh <-chan struct{} 1274 } 1275 1276 type httpError struct { 1277 err string 1278 timeout bool 1279 } 1280 1281 func (e *httpError) Error() string { return e.err } 1282 func (e *httpError) Timeout() bool { return e.timeout } 1283 func (e *httpError) Temporary() bool { return true } 1284 1285 var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} 1286 var errClosed error = &httpError{err: "net/http: transport closed before response was received"} 1287 var errRequestCanceled = errors.New("net/http: request canceled") 1288 1289 func nop() {} 1290 1291 // testHooks. Always non-nil. 1292 var ( 1293 testHookEnterRoundTrip = nop 1294 testHookWaitResLoop = nop 1295 testHookRoundTripRetried = nop 1296 testHookPrePendingDial = nop 1297 testHookPostPendingDial = nop 1298 1299 testHookMu sync.Locker = fakeLocker{} // guards following 1300 testHookReadLoopBeforeNextRead = nop 1301 ) 1302 1303 // beforeRespHeaderError is used to indicate when an IO error has occurred before 1304 // any header data was received. 1305 type beforeRespHeaderError struct { 1306 error 1307 } 1308 1309 func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 1310 testHookEnterRoundTrip() 1311 if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { 1312 pc.t.putIdleConn(pc) 1313 return nil, errRequestCanceled 1314 } 1315 pc.lk.Lock() 1316 pc.numExpectedResponses++ 1317 headerFn := pc.mutateHeaderFunc 1318 pc.lk.Unlock() 1319 1320 if headerFn != nil { 1321 headerFn(req.extraHeaders()) 1322 } 1323 1324 // Ask for a compressed version if the caller didn't set their 1325 // own value for Accept-Encoding. We only attempt to 1326 // uncompress the gzip stream if we were the layer that 1327 // requested it. 1328 requestedGzip := false 1329 if !pc.t.DisableCompression && 1330 req.Header.Get("Accept-Encoding") == "" && 1331 req.Header.Get("Range") == "" && 1332 req.Method != "HEAD" { 1333 // Request gzip only, not deflate. Deflate is ambiguous and 1334 // not as universally supported anyway. 1335 // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 1336 // 1337 // Note that we don't request this for HEAD requests, 1338 // due to a bug in nginx: 1339 // http://trac.nginx.org/nginx/ticket/358 1340 // https://golang.org/issue/5522 1341 // 1342 // We don't request gzip if the request is for a range, since 1343 // auto-decoding a portion of a gzipped document will just fail 1344 // anyway. See https://golang.org/issue/8923 1345 requestedGzip = true 1346 req.extraHeaders().Set("Accept-Encoding", "gzip") 1347 } 1348 1349 var continueCh chan struct{} 1350 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 1351 continueCh = make(chan struct{}, 1) 1352 } 1353 1354 if pc.t.DisableKeepAlives { 1355 req.extraHeaders().Set("Connection", "close") 1356 } 1357 1358 // Write the request concurrently with waiting for a response, 1359 // in case the server decides to reply before reading our full 1360 // request body. 1361 writeErrCh := make(chan error, 1) 1362 pc.writech <- writeRequest{req, writeErrCh, continueCh} 1363 1364 resc := make(chan responseAndError, 1) 1365 pc.reqch <- requestAndChan{req.Request, resc, requestedGzip, continueCh} 1366 1367 var re responseAndError 1368 var respHeaderTimer <-chan time.Time 1369 cancelChan := req.Request.Cancel 1370 WaitResponse: 1371 for { 1372 testHookWaitResLoop() 1373 select { 1374 case err := <-writeErrCh: 1375 if isNetWriteError(err) { 1376 // Issue 11745. If we failed to write the request 1377 // body, it's possible the server just heard enough 1378 // and already wrote to us. Prioritize the server's 1379 // response over returning a body write error. 1380 select { 1381 case re = <-resc: 1382 pc.close() 1383 break WaitResponse 1384 case <-time.After(50 * time.Millisecond): 1385 // Fall through. 1386 } 1387 } 1388 if err != nil { 1389 re = responseAndError{err: beforeRespHeaderError{err}} 1390 pc.close() 1391 break WaitResponse 1392 } 1393 if d := pc.t.ResponseHeaderTimeout; d > 0 { 1394 timer := time.NewTimer(d) 1395 defer timer.Stop() // prevent leaks 1396 respHeaderTimer = timer.C 1397 } 1398 case <-pc.closech: 1399 var err error 1400 if pc.isCanceled() { 1401 err = errRequestCanceled 1402 } else { 1403 err = beforeRespHeaderError{errClosed} 1404 } 1405 re = responseAndError{err: err} 1406 break WaitResponse 1407 case <-respHeaderTimer: 1408 pc.close() 1409 re = responseAndError{err: errTimeout} 1410 break WaitResponse 1411 case re = <-resc: 1412 break WaitResponse 1413 case <-cancelChan: 1414 pc.t.CancelRequest(req.Request) 1415 cancelChan = nil 1416 } 1417 } 1418 1419 if re.err != nil { 1420 pc.t.setReqCanceler(req.Request, nil) 1421 } 1422 return re.res(), re.err 1423 } 1424 1425 // markBroken marks a connection as broken (so it's not reused). 1426 // It differs from close in that it doesn't close the underlying 1427 // connection for use when it's still being read. 1428 func (pc *persistConn) markBroken() { 1429 pc.lk.Lock() 1430 defer pc.lk.Unlock() 1431 pc.broken = true 1432 } 1433 1434 // markReused marks this connection as having been successfully used for a 1435 // request and response. 1436 func (pc *persistConn) markReused() { 1437 pc.lk.Lock() 1438 pc.reused = true 1439 pc.lk.Unlock() 1440 } 1441 1442 func (pc *persistConn) close() { 1443 pc.lk.Lock() 1444 defer pc.lk.Unlock() 1445 pc.closeLocked() 1446 } 1447 1448 func (pc *persistConn) closeLocked() { 1449 pc.broken = true 1450 if !pc.closed { 1451 pc.conn.Close() 1452 pc.closed = true 1453 close(pc.closech) 1454 } 1455 pc.mutateHeaderFunc = nil 1456 } 1457 1458 var portMap = map[string]string{ 1459 "http": "80", 1460 "https": "443", 1461 } 1462 1463 // canonicalAddr returns url.Host but always with a ":port" suffix 1464 func canonicalAddr(url *url.URL) string { 1465 addr := url.Host 1466 if !hasPort(addr) { 1467 return addr + ":" + portMap[url.Scheme] 1468 } 1469 return addr 1470 } 1471 1472 // bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most 1473 // once, right before its final (error-producing) Read or Close call 1474 // returns. fn should return the new error to return from Read or Close. 1475 // 1476 // If earlyCloseFn is non-nil and Close is called before io.EOF is 1477 // seen, earlyCloseFn is called instead of fn, and its return value is 1478 // the return value from Close. 1479 type bodyEOFSignal struct { 1480 body io.ReadCloser 1481 mu sync.Mutex // guards following 4 fields 1482 closed bool // whether Close has been called 1483 rerr error // sticky Read error 1484 fn func(error) error // err will be nil on Read io.EOF 1485 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 1486 } 1487 1488 func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 1489 es.mu.Lock() 1490 closed, rerr := es.closed, es.rerr 1491 es.mu.Unlock() 1492 if closed { 1493 return 0, errors.New("http: read on closed response body") 1494 } 1495 if rerr != nil { 1496 return 0, rerr 1497 } 1498 1499 n, err = es.body.Read(p) 1500 if err != nil { 1501 es.mu.Lock() 1502 defer es.mu.Unlock() 1503 if es.rerr == nil { 1504 es.rerr = err 1505 } 1506 err = es.condfn(err) 1507 } 1508 return 1509 } 1510 1511 func (es *bodyEOFSignal) Close() error { 1512 es.mu.Lock() 1513 defer es.mu.Unlock() 1514 if es.closed { 1515 return nil 1516 } 1517 es.closed = true 1518 if es.earlyCloseFn != nil && es.rerr != io.EOF { 1519 return es.earlyCloseFn() 1520 } 1521 err := es.body.Close() 1522 return es.condfn(err) 1523 } 1524 1525 // caller must hold es.mu. 1526 func (es *bodyEOFSignal) condfn(err error) error { 1527 if es.fn == nil { 1528 return err 1529 } 1530 err = es.fn(err) 1531 es.fn = nil 1532 return err 1533 } 1534 1535 // gzipReader wraps a response body so it can lazily 1536 // call gzip.NewReader on the first call to Read 1537 type gzipReader struct { 1538 body io.ReadCloser // underlying Response.Body 1539 zr io.Reader // lazily-initialized gzip reader 1540 } 1541 1542 func (gz *gzipReader) Read(p []byte) (n int, err error) { 1543 if gz.zr == nil { 1544 gz.zr, err = gzip.NewReader(gz.body) 1545 if err != nil { 1546 return 0, err 1547 } 1548 } 1549 return gz.zr.Read(p) 1550 } 1551 1552 func (gz *gzipReader) Close() error { 1553 return gz.body.Close() 1554 } 1555 1556 type readerAndCloser struct { 1557 io.Reader 1558 io.Closer 1559 } 1560 1561 type tlsHandshakeTimeoutError struct{} 1562 1563 func (tlsHandshakeTimeoutError) Timeout() bool { return true } 1564 func (tlsHandshakeTimeoutError) Temporary() bool { return true } 1565 func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 1566 1567 type noteEOFReader struct { 1568 r io.Reader 1569 sawEOF *bool 1570 } 1571 1572 func (nr noteEOFReader) Read(p []byte) (n int, err error) { 1573 n, err = nr.r.Read(p) 1574 if err == io.EOF { 1575 *nr.sawEOF = true 1576 } 1577 return 1578 } 1579 1580 // fakeLocker is a sync.Locker which does nothing. It's used to guard 1581 // test-only fields when not under test, to avoid runtime atomic 1582 // overhead. 1583 type fakeLocker struct{} 1584 1585 func (fakeLocker) Lock() {} 1586 func (fakeLocker) Unlock() {} 1587 1588 func isNetWriteError(err error) bool { 1589 switch e := err.(type) { 1590 case *url.Error: 1591 return isNetWriteError(e.Err) 1592 case *net.OpError: 1593 return e.Op == "write" 1594 default: 1595 return false 1596 } 1597 } 1598 1599 // cloneTLSConfig returns a shallow clone of the exported 1600 // fields of cfg, ignoring the unexported sync.Once, which 1601 // contains a mutex and must not be copied. 1602 // 1603 // The cfg must not be in active use by tls.Server, or else 1604 // there can still be a race with tls.Server updating SessionTicketKey 1605 // and our copying it, and also a race with the server setting 1606 // SessionTicketsDisabled=false on failure to set the random 1607 // ticket key. 1608 // 1609 // If cfg is nil, a new zero tls.Config is returned. 1610 func cloneTLSConfig(cfg *tls.Config) *tls.Config { 1611 if cfg == nil { 1612 return &tls.Config{} 1613 } 1614 return &tls.Config{ 1615 Rand: cfg.Rand, 1616 Time: cfg.Time, 1617 Certificates: cfg.Certificates, 1618 NameToCertificate: cfg.NameToCertificate, 1619 GetCertificate: cfg.GetCertificate, 1620 RootCAs: cfg.RootCAs, 1621 NextProtos: cfg.NextProtos, 1622 ServerName: cfg.ServerName, 1623 ClientAuth: cfg.ClientAuth, 1624 ClientCAs: cfg.ClientCAs, 1625 InsecureSkipVerify: cfg.InsecureSkipVerify, 1626 CipherSuites: cfg.CipherSuites, 1627 PreferServerCipherSuites: cfg.PreferServerCipherSuites, 1628 SessionTicketsDisabled: cfg.SessionTicketsDisabled, 1629 SessionTicketKey: cfg.SessionTicketKey, 1630 ClientSessionCache: cfg.ClientSessionCache, 1631 MinVersion: cfg.MinVersion, 1632 MaxVersion: cfg.MaxVersion, 1633 CurvePreferences: cfg.CurvePreferences, 1634 } 1635 } 1636 1637 // cloneTLSClientConfig is like cloneTLSConfig but omits 1638 // the fields SessionTicketsDisabled and SessionTicketKey. 1639 // This makes it safe to call cloneTLSClientConfig on a config 1640 // in active use by a server. 1641 func cloneTLSClientConfig(cfg *tls.Config) *tls.Config { 1642 if cfg == nil { 1643 return &tls.Config{} 1644 } 1645 return &tls.Config{ 1646 Rand: cfg.Rand, 1647 Time: cfg.Time, 1648 Certificates: cfg.Certificates, 1649 NameToCertificate: cfg.NameToCertificate, 1650 GetCertificate: cfg.GetCertificate, 1651 RootCAs: cfg.RootCAs, 1652 NextProtos: cfg.NextProtos, 1653 ServerName: cfg.ServerName, 1654 ClientAuth: cfg.ClientAuth, 1655 ClientCAs: cfg.ClientCAs, 1656 InsecureSkipVerify: cfg.InsecureSkipVerify, 1657 CipherSuites: cfg.CipherSuites, 1658 PreferServerCipherSuites: cfg.PreferServerCipherSuites, 1659 ClientSessionCache: cfg.ClientSessionCache, 1660 MinVersion: cfg.MinVersion, 1661 MaxVersion: cfg.MaxVersion, 1662 CurvePreferences: cfg.CurvePreferences, 1663 } 1664 }