github.com/dannin/go@v0.0.0-20161031215817-d35dfd405eaa/src/net/http/transport_test.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Tests for transport.go. 6 // 7 // More tests are in clientserver_test.go (for things testing both client & server for both 8 // HTTP/1 and HTTP/2). This 9 10 package http_test 11 12 import ( 13 "bufio" 14 "bytes" 15 "compress/gzip" 16 "context" 17 "crypto/rand" 18 "crypto/tls" 19 "errors" 20 "fmt" 21 "internal/nettrace" 22 "internal/testenv" 23 "io" 24 "io/ioutil" 25 "log" 26 "net" 27 . "net/http" 28 "net/http/httptest" 29 "net/http/httptrace" 30 "net/http/httputil" 31 "net/http/internal" 32 "net/url" 33 "os" 34 "reflect" 35 "runtime" 36 "strconv" 37 "strings" 38 "sync" 39 "testing" 40 "time" 41 ) 42 43 // TODO: test 5 pipelined requests with responses: 1) OK, 2) OK, Connection: Close 44 // and then verify that the final 2 responses get errors back. 45 46 // hostPortHandler writes back the client's "host:port". 47 var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) { 48 if r.FormValue("close") == "true" { 49 w.Header().Set("Connection", "close") 50 } 51 w.Header().Set("X-Saw-Close", fmt.Sprint(r.Close)) 52 w.Write([]byte(r.RemoteAddr)) 53 }) 54 55 // testCloseConn is a net.Conn tracked by a testConnSet. 56 type testCloseConn struct { 57 net.Conn 58 set *testConnSet 59 } 60 61 func (c *testCloseConn) Close() error { 62 c.set.remove(c) 63 return c.Conn.Close() 64 } 65 66 // testConnSet tracks a set of TCP connections and whether they've 67 // been closed. 68 type testConnSet struct { 69 t *testing.T 70 mu sync.Mutex // guards closed and list 71 closed map[net.Conn]bool 72 list []net.Conn // in order created 73 } 74 75 func (tcs *testConnSet) insert(c net.Conn) { 76 tcs.mu.Lock() 77 defer tcs.mu.Unlock() 78 tcs.closed[c] = false 79 tcs.list = append(tcs.list, c) 80 } 81 82 func (tcs *testConnSet) remove(c net.Conn) { 83 tcs.mu.Lock() 84 defer tcs.mu.Unlock() 85 tcs.closed[c] = true 86 } 87 88 // some tests use this to manage raw tcp connections for later inspection 89 func makeTestDial(t *testing.T) (*testConnSet, func(n, addr string) (net.Conn, error)) { 90 connSet := &testConnSet{ 91 t: t, 92 closed: make(map[net.Conn]bool), 93 } 94 dial := func(n, addr string) (net.Conn, error) { 95 c, err := net.Dial(n, addr) 96 if err != nil { 97 return nil, err 98 } 99 tc := &testCloseConn{c, connSet} 100 connSet.insert(tc) 101 return tc, nil 102 } 103 return connSet, dial 104 } 105 106 func (tcs *testConnSet) check(t *testing.T) { 107 tcs.mu.Lock() 108 defer tcs.mu.Unlock() 109 for i := 4; i >= 0; i-- { 110 for i, c := range tcs.list { 111 if tcs.closed[c] { 112 continue 113 } 114 if i != 0 { 115 tcs.mu.Unlock() 116 time.Sleep(50 * time.Millisecond) 117 tcs.mu.Lock() 118 continue 119 } 120 t.Errorf("TCP connection #%d, %p (of %d total) was not closed", i+1, c, len(tcs.list)) 121 } 122 } 123 } 124 125 // Two subsequent requests and verify their response is the same. 126 // The response from the server is our own IP:port 127 func TestTransportKeepAlives(t *testing.T) { 128 defer afterTest(t) 129 ts := httptest.NewServer(hostPortHandler) 130 defer ts.Close() 131 132 for _, disableKeepAlive := range []bool{false, true} { 133 tr := &Transport{DisableKeepAlives: disableKeepAlive} 134 defer tr.CloseIdleConnections() 135 c := &Client{Transport: tr} 136 137 fetch := func(n int) string { 138 res, err := c.Get(ts.URL) 139 if err != nil { 140 t.Fatalf("error in disableKeepAlive=%v, req #%d, GET: %v", disableKeepAlive, n, err) 141 } 142 body, err := ioutil.ReadAll(res.Body) 143 if err != nil { 144 t.Fatalf("error in disableKeepAlive=%v, req #%d, ReadAll: %v", disableKeepAlive, n, err) 145 } 146 return string(body) 147 } 148 149 body1 := fetch(1) 150 body2 := fetch(2) 151 152 bodiesDiffer := body1 != body2 153 if bodiesDiffer != disableKeepAlive { 154 t.Errorf("error in disableKeepAlive=%v. unexpected bodiesDiffer=%v; body1=%q; body2=%q", 155 disableKeepAlive, bodiesDiffer, body1, body2) 156 } 157 } 158 } 159 160 func TestTransportConnectionCloseOnResponse(t *testing.T) { 161 defer afterTest(t) 162 ts := httptest.NewServer(hostPortHandler) 163 defer ts.Close() 164 165 connSet, testDial := makeTestDial(t) 166 167 for _, connectionClose := range []bool{false, true} { 168 tr := &Transport{ 169 Dial: testDial, 170 } 171 c := &Client{Transport: tr} 172 173 fetch := func(n int) string { 174 req := new(Request) 175 var err error 176 req.URL, err = url.Parse(ts.URL + fmt.Sprintf("/?close=%v", connectionClose)) 177 if err != nil { 178 t.Fatalf("URL parse error: %v", err) 179 } 180 req.Method = "GET" 181 req.Proto = "HTTP/1.1" 182 req.ProtoMajor = 1 183 req.ProtoMinor = 1 184 185 res, err := c.Do(req) 186 if err != nil { 187 t.Fatalf("error in connectionClose=%v, req #%d, Do: %v", connectionClose, n, err) 188 } 189 defer res.Body.Close() 190 body, err := ioutil.ReadAll(res.Body) 191 if err != nil { 192 t.Fatalf("error in connectionClose=%v, req #%d, ReadAll: %v", connectionClose, n, err) 193 } 194 return string(body) 195 } 196 197 body1 := fetch(1) 198 body2 := fetch(2) 199 bodiesDiffer := body1 != body2 200 if bodiesDiffer != connectionClose { 201 t.Errorf("error in connectionClose=%v. unexpected bodiesDiffer=%v; body1=%q; body2=%q", 202 connectionClose, bodiesDiffer, body1, body2) 203 } 204 205 tr.CloseIdleConnections() 206 } 207 208 connSet.check(t) 209 } 210 211 func TestTransportConnectionCloseOnRequest(t *testing.T) { 212 defer afterTest(t) 213 ts := httptest.NewServer(hostPortHandler) 214 defer ts.Close() 215 216 connSet, testDial := makeTestDial(t) 217 218 for _, connectionClose := range []bool{false, true} { 219 tr := &Transport{ 220 Dial: testDial, 221 } 222 c := &Client{Transport: tr} 223 224 fetch := func(n int) string { 225 req := new(Request) 226 var err error 227 req.URL, err = url.Parse(ts.URL) 228 if err != nil { 229 t.Fatalf("URL parse error: %v", err) 230 } 231 req.Method = "GET" 232 req.Proto = "HTTP/1.1" 233 req.ProtoMajor = 1 234 req.ProtoMinor = 1 235 req.Close = connectionClose 236 237 res, err := c.Do(req) 238 if err != nil { 239 t.Fatalf("error in connectionClose=%v, req #%d, Do: %v", connectionClose, n, err) 240 } 241 if got, want := res.Header.Get("X-Saw-Close"), fmt.Sprint(connectionClose); got != want { 242 t.Errorf("For connectionClose = %v; handler's X-Saw-Close was %v; want %v", 243 connectionClose, got, !connectionClose) 244 } 245 body, err := ioutil.ReadAll(res.Body) 246 if err != nil { 247 t.Fatalf("error in connectionClose=%v, req #%d, ReadAll: %v", connectionClose, n, err) 248 } 249 return string(body) 250 } 251 252 body1 := fetch(1) 253 body2 := fetch(2) 254 bodiesDiffer := body1 != body2 255 if bodiesDiffer != connectionClose { 256 t.Errorf("error in connectionClose=%v. unexpected bodiesDiffer=%v; body1=%q; body2=%q", 257 connectionClose, bodiesDiffer, body1, body2) 258 } 259 260 tr.CloseIdleConnections() 261 } 262 263 connSet.check(t) 264 } 265 266 // if the Transport's DisableKeepAlives is set, all requests should 267 // send Connection: close. 268 // HTTP/1-only (Connection: close doesn't exist in h2) 269 func TestTransportConnectionCloseOnRequestDisableKeepAlive(t *testing.T) { 270 defer afterTest(t) 271 ts := httptest.NewServer(hostPortHandler) 272 defer ts.Close() 273 274 tr := &Transport{ 275 DisableKeepAlives: true, 276 } 277 c := &Client{Transport: tr} 278 res, err := c.Get(ts.URL) 279 if err != nil { 280 t.Fatal(err) 281 } 282 res.Body.Close() 283 if res.Header.Get("X-Saw-Close") != "true" { 284 t.Errorf("handler didn't see Connection: close ") 285 } 286 } 287 288 func TestTransportIdleCacheKeys(t *testing.T) { 289 defer afterTest(t) 290 ts := httptest.NewServer(hostPortHandler) 291 defer ts.Close() 292 293 tr := &Transport{DisableKeepAlives: false} 294 c := &Client{Transport: tr} 295 296 if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { 297 t.Errorf("After CloseIdleConnections expected %d idle conn cache keys; got %d", e, g) 298 } 299 300 resp, err := c.Get(ts.URL) 301 if err != nil { 302 t.Error(err) 303 } 304 ioutil.ReadAll(resp.Body) 305 306 keys := tr.IdleConnKeysForTesting() 307 if e, g := 1, len(keys); e != g { 308 t.Fatalf("After Get expected %d idle conn cache keys; got %d", e, g) 309 } 310 311 if e := "|http|" + ts.Listener.Addr().String(); keys[0] != e { 312 t.Errorf("Expected idle cache key %q; got %q", e, keys[0]) 313 } 314 315 tr.CloseIdleConnections() 316 if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { 317 t.Errorf("After CloseIdleConnections expected %d idle conn cache keys; got %d", e, g) 318 } 319 } 320 321 // Tests that the HTTP transport re-uses connections when a client 322 // reads to the end of a response Body without closing it. 323 func TestTransportReadToEndReusesConn(t *testing.T) { 324 defer afterTest(t) 325 const msg = "foobar" 326 327 var addrSeen map[string]int 328 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 329 addrSeen[r.RemoteAddr]++ 330 if r.URL.Path == "/chunked/" { 331 w.WriteHeader(200) 332 w.(Flusher).Flush() 333 } else { 334 w.Header().Set("Content-Type", strconv.Itoa(len(msg))) 335 w.WriteHeader(200) 336 } 337 w.Write([]byte(msg)) 338 })) 339 defer ts.Close() 340 341 buf := make([]byte, len(msg)) 342 343 for pi, path := range []string{"/content-length/", "/chunked/"} { 344 wantLen := []int{len(msg), -1}[pi] 345 addrSeen = make(map[string]int) 346 for i := 0; i < 3; i++ { 347 res, err := Get(ts.URL + path) 348 if err != nil { 349 t.Errorf("Get %s: %v", path, err) 350 continue 351 } 352 // We want to close this body eventually (before the 353 // defer afterTest at top runs), but not before the 354 // len(addrSeen) check at the bottom of this test, 355 // since Closing this early in the loop would risk 356 // making connections be re-used for the wrong reason. 357 defer res.Body.Close() 358 359 if res.ContentLength != int64(wantLen) { 360 t.Errorf("%s res.ContentLength = %d; want %d", path, res.ContentLength, wantLen) 361 } 362 n, err := res.Body.Read(buf) 363 if n != len(msg) || err != io.EOF { 364 t.Errorf("%s Read = %v, %v; want %d, EOF", path, n, err, len(msg)) 365 } 366 } 367 if len(addrSeen) != 1 { 368 t.Errorf("for %s, server saw %d distinct client addresses; want 1", path, len(addrSeen)) 369 } 370 } 371 } 372 373 func TestTransportMaxPerHostIdleConns(t *testing.T) { 374 defer afterTest(t) 375 resch := make(chan string) 376 gotReq := make(chan bool) 377 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 378 gotReq <- true 379 msg := <-resch 380 _, err := w.Write([]byte(msg)) 381 if err != nil { 382 t.Fatalf("Write: %v", err) 383 } 384 })) 385 defer ts.Close() 386 maxIdleConnsPerHost := 2 387 tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConnsPerHost} 388 c := &Client{Transport: tr} 389 390 // Start 3 outstanding requests and wait for the server to get them. 391 // Their responses will hang until we write to resch, though. 392 donech := make(chan bool) 393 doReq := func() { 394 resp, err := c.Get(ts.URL) 395 if err != nil { 396 t.Error(err) 397 return 398 } 399 if _, err := ioutil.ReadAll(resp.Body); err != nil { 400 t.Errorf("ReadAll: %v", err) 401 return 402 } 403 donech <- true 404 } 405 go doReq() 406 <-gotReq 407 go doReq() 408 <-gotReq 409 go doReq() 410 <-gotReq 411 412 if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { 413 t.Fatalf("Before writes, expected %d idle conn cache keys; got %d", e, g) 414 } 415 416 resch <- "res1" 417 <-donech 418 keys := tr.IdleConnKeysForTesting() 419 if e, g := 1, len(keys); e != g { 420 t.Fatalf("after first response, expected %d idle conn cache keys; got %d", e, g) 421 } 422 cacheKey := "|http|" + ts.Listener.Addr().String() 423 if keys[0] != cacheKey { 424 t.Fatalf("Expected idle cache key %q; got %q", cacheKey, keys[0]) 425 } 426 if e, g := 1, tr.IdleConnCountForTesting(cacheKey); e != g { 427 t.Errorf("after first response, expected %d idle conns; got %d", e, g) 428 } 429 430 resch <- "res2" 431 <-donech 432 if g, w := tr.IdleConnCountForTesting(cacheKey), 2; g != w { 433 t.Errorf("after second response, idle conns = %d; want %d", g, w) 434 } 435 436 resch <- "res3" 437 <-donech 438 if g, w := tr.IdleConnCountForTesting(cacheKey), maxIdleConnsPerHost; g != w { 439 t.Errorf("after third response, idle conns = %d; want %d", g, w) 440 } 441 } 442 443 func TestTransportRemovesDeadIdleConnections(t *testing.T) { 444 defer afterTest(t) 445 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 446 io.WriteString(w, r.RemoteAddr) 447 })) 448 defer ts.Close() 449 450 tr := &Transport{} 451 defer tr.CloseIdleConnections() 452 c := &Client{Transport: tr} 453 454 doReq := func(name string) string { 455 // Do a POST instead of a GET to prevent the Transport's 456 // idempotent request retry logic from kicking in... 457 res, err := c.Post(ts.URL, "", nil) 458 if err != nil { 459 t.Fatalf("%s: %v", name, err) 460 } 461 if res.StatusCode != 200 { 462 t.Fatalf("%s: %v", name, res.Status) 463 } 464 defer res.Body.Close() 465 slurp, err := ioutil.ReadAll(res.Body) 466 if err != nil { 467 t.Fatalf("%s: %v", name, err) 468 } 469 return string(slurp) 470 } 471 472 first := doReq("first") 473 keys1 := tr.IdleConnKeysForTesting() 474 475 ts.CloseClientConnections() 476 477 var keys2 []string 478 if !waitCondition(3*time.Second, 50*time.Millisecond, func() bool { 479 keys2 = tr.IdleConnKeysForTesting() 480 return len(keys2) == 0 481 }) { 482 t.Fatalf("Transport didn't notice idle connection's death.\nbefore: %q\n after: %q\n", keys1, keys2) 483 } 484 485 second := doReq("second") 486 if first == second { 487 t.Errorf("expected a different connection between requests. got %q both times", first) 488 } 489 } 490 491 func TestTransportServerClosingUnexpectedly(t *testing.T) { 492 setParallel(t) 493 defer afterTest(t) 494 ts := httptest.NewServer(hostPortHandler) 495 defer ts.Close() 496 497 tr := &Transport{} 498 c := &Client{Transport: tr} 499 500 fetch := func(n, retries int) string { 501 condFatalf := func(format string, arg ...interface{}) { 502 if retries <= 0 { 503 t.Fatalf(format, arg...) 504 } 505 t.Logf("retrying shortly after expected error: "+format, arg...) 506 time.Sleep(time.Second / time.Duration(retries)) 507 } 508 for retries >= 0 { 509 retries-- 510 res, err := c.Get(ts.URL) 511 if err != nil { 512 condFatalf("error in req #%d, GET: %v", n, err) 513 continue 514 } 515 body, err := ioutil.ReadAll(res.Body) 516 if err != nil { 517 condFatalf("error in req #%d, ReadAll: %v", n, err) 518 continue 519 } 520 res.Body.Close() 521 return string(body) 522 } 523 panic("unreachable") 524 } 525 526 body1 := fetch(1, 0) 527 body2 := fetch(2, 0) 528 529 ts.CloseClientConnections() // surprise! 530 531 // This test has an expected race. Sleeping for 25 ms prevents 532 // it on most fast machines, causing the next fetch() call to 533 // succeed quickly. But if we do get errors, fetch() will retry 5 534 // times with some delays between. 535 time.Sleep(25 * time.Millisecond) 536 537 body3 := fetch(3, 5) 538 539 if body1 != body2 { 540 t.Errorf("expected body1 and body2 to be equal") 541 } 542 if body2 == body3 { 543 t.Errorf("expected body2 and body3 to be different") 544 } 545 } 546 547 // Test for https://golang.org/issue/2616 (appropriate issue number) 548 // This fails pretty reliably with GOMAXPROCS=100 or something high. 549 func TestStressSurpriseServerCloses(t *testing.T) { 550 defer afterTest(t) 551 if testing.Short() { 552 t.Skip("skipping test in short mode") 553 } 554 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 555 w.Header().Set("Content-Length", "5") 556 w.Header().Set("Content-Type", "text/plain") 557 w.Write([]byte("Hello")) 558 w.(Flusher).Flush() 559 conn, buf, _ := w.(Hijacker).Hijack() 560 buf.Flush() 561 conn.Close() 562 })) 563 defer ts.Close() 564 565 tr := &Transport{DisableKeepAlives: false} 566 c := &Client{Transport: tr} 567 defer tr.CloseIdleConnections() 568 569 // Do a bunch of traffic from different goroutines. Send to activityc 570 // after each request completes, regardless of whether it failed. 571 // If these are too high, OS X exhausts its ephemeral ports 572 // and hangs waiting for them to transition TCP states. That's 573 // not what we want to test. TODO(bradfitz): use an io.Pipe 574 // dialer for this test instead? 575 const ( 576 numClients = 20 577 reqsPerClient = 25 578 ) 579 activityc := make(chan bool) 580 for i := 0; i < numClients; i++ { 581 go func() { 582 for i := 0; i < reqsPerClient; i++ { 583 res, err := c.Get(ts.URL) 584 if err == nil { 585 // We expect errors since the server is 586 // hanging up on us after telling us to 587 // send more requests, so we don't 588 // actually care what the error is. 589 // But we want to close the body in cases 590 // where we won the race. 591 res.Body.Close() 592 } 593 activityc <- true 594 } 595 }() 596 } 597 598 // Make sure all the request come back, one way or another. 599 for i := 0; i < numClients*reqsPerClient; i++ { 600 select { 601 case <-activityc: 602 case <-time.After(5 * time.Second): 603 t.Fatalf("presumed deadlock; no HTTP client activity seen in awhile") 604 } 605 } 606 } 607 608 // TestTransportHeadResponses verifies that we deal with Content-Lengths 609 // with no bodies properly 610 func TestTransportHeadResponses(t *testing.T) { 611 defer afterTest(t) 612 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 613 if r.Method != "HEAD" { 614 panic("expected HEAD; got " + r.Method) 615 } 616 w.Header().Set("Content-Length", "123") 617 w.WriteHeader(200) 618 })) 619 defer ts.Close() 620 621 tr := &Transport{DisableKeepAlives: false} 622 c := &Client{Transport: tr} 623 for i := 0; i < 2; i++ { 624 res, err := c.Head(ts.URL) 625 if err != nil { 626 t.Errorf("error on loop %d: %v", i, err) 627 continue 628 } 629 if e, g := "123", res.Header.Get("Content-Length"); e != g { 630 t.Errorf("loop %d: expected Content-Length header of %q, got %q", i, e, g) 631 } 632 if e, g := int64(123), res.ContentLength; e != g { 633 t.Errorf("loop %d: expected res.ContentLength of %v, got %v", i, e, g) 634 } 635 if all, err := ioutil.ReadAll(res.Body); err != nil { 636 t.Errorf("loop %d: Body ReadAll: %v", i, err) 637 } else if len(all) != 0 { 638 t.Errorf("Bogus body %q", all) 639 } 640 } 641 } 642 643 // TestTransportHeadChunkedResponse verifies that we ignore chunked transfer-encoding 644 // on responses to HEAD requests. 645 func TestTransportHeadChunkedResponse(t *testing.T) { 646 defer afterTest(t) 647 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 648 if r.Method != "HEAD" { 649 panic("expected HEAD; got " + r.Method) 650 } 651 w.Header().Set("Transfer-Encoding", "chunked") // client should ignore 652 w.Header().Set("x-client-ipport", r.RemoteAddr) 653 w.WriteHeader(200) 654 })) 655 defer ts.Close() 656 657 tr := &Transport{DisableKeepAlives: false} 658 c := &Client{Transport: tr} 659 defer tr.CloseIdleConnections() 660 661 // Ensure that we wait for the readLoop to complete before 662 // calling Head again 663 didRead := make(chan bool) 664 SetReadLoopBeforeNextReadHook(func() { didRead <- true }) 665 defer SetReadLoopBeforeNextReadHook(nil) 666 667 res1, err := c.Head(ts.URL) 668 <-didRead 669 670 if err != nil { 671 t.Fatalf("request 1 error: %v", err) 672 } 673 674 res2, err := c.Head(ts.URL) 675 <-didRead 676 677 if err != nil { 678 t.Fatalf("request 2 error: %v", err) 679 } 680 if v1, v2 := res1.Header.Get("x-client-ipport"), res2.Header.Get("x-client-ipport"); v1 != v2 { 681 t.Errorf("ip/ports differed between head requests: %q vs %q", v1, v2) 682 } 683 } 684 685 var roundTripTests = []struct { 686 accept string 687 expectAccept string 688 compressed bool 689 }{ 690 // Requests with no accept-encoding header use transparent compression 691 {"", "gzip", false}, 692 // Requests with other accept-encoding should pass through unmodified 693 {"foo", "foo", false}, 694 // Requests with accept-encoding == gzip should be passed through 695 {"gzip", "gzip", true}, 696 } 697 698 // Test that the modification made to the Request by the RoundTripper is cleaned up 699 func TestRoundTripGzip(t *testing.T) { 700 defer afterTest(t) 701 const responseBody = "test response body" 702 ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { 703 accept := req.Header.Get("Accept-Encoding") 704 if expect := req.FormValue("expect_accept"); accept != expect { 705 t.Errorf("in handler, test %v: Accept-Encoding = %q, want %q", 706 req.FormValue("testnum"), accept, expect) 707 } 708 if accept == "gzip" { 709 rw.Header().Set("Content-Encoding", "gzip") 710 gz := gzip.NewWriter(rw) 711 gz.Write([]byte(responseBody)) 712 gz.Close() 713 } else { 714 rw.Header().Set("Content-Encoding", accept) 715 rw.Write([]byte(responseBody)) 716 } 717 })) 718 defer ts.Close() 719 720 for i, test := range roundTripTests { 721 // Test basic request (no accept-encoding) 722 req, _ := NewRequest("GET", fmt.Sprintf("%s/?testnum=%d&expect_accept=%s", ts.URL, i, test.expectAccept), nil) 723 if test.accept != "" { 724 req.Header.Set("Accept-Encoding", test.accept) 725 } 726 res, err := DefaultTransport.RoundTrip(req) 727 var body []byte 728 if test.compressed { 729 var r *gzip.Reader 730 r, err = gzip.NewReader(res.Body) 731 if err != nil { 732 t.Errorf("%d. gzip NewReader: %v", i, err) 733 continue 734 } 735 body, err = ioutil.ReadAll(r) 736 res.Body.Close() 737 } else { 738 body, err = ioutil.ReadAll(res.Body) 739 } 740 if err != nil { 741 t.Errorf("%d. Error: %q", i, err) 742 continue 743 } 744 if g, e := string(body), responseBody; g != e { 745 t.Errorf("%d. body = %q; want %q", i, g, e) 746 } 747 if g, e := req.Header.Get("Accept-Encoding"), test.accept; g != e { 748 t.Errorf("%d. Accept-Encoding = %q; want %q (it was mutated, in violation of RoundTrip contract)", i, g, e) 749 } 750 if g, e := res.Header.Get("Content-Encoding"), test.accept; g != e { 751 t.Errorf("%d. Content-Encoding = %q; want %q", i, g, e) 752 } 753 } 754 755 } 756 757 func TestTransportGzip(t *testing.T) { 758 defer afterTest(t) 759 const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" 760 const nRandBytes = 1024 * 1024 761 ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { 762 if req.Method == "HEAD" { 763 if g := req.Header.Get("Accept-Encoding"); g != "" { 764 t.Errorf("HEAD request sent with Accept-Encoding of %q; want none", g) 765 } 766 return 767 } 768 if g, e := req.Header.Get("Accept-Encoding"), "gzip"; g != e { 769 t.Errorf("Accept-Encoding = %q, want %q", g, e) 770 } 771 rw.Header().Set("Content-Encoding", "gzip") 772 773 var w io.Writer = rw 774 var buf bytes.Buffer 775 if req.FormValue("chunked") == "0" { 776 w = &buf 777 defer io.Copy(rw, &buf) 778 defer func() { 779 rw.Header().Set("Content-Length", strconv.Itoa(buf.Len())) 780 }() 781 } 782 gz := gzip.NewWriter(w) 783 gz.Write([]byte(testString)) 784 if req.FormValue("body") == "large" { 785 io.CopyN(gz, rand.Reader, nRandBytes) 786 } 787 gz.Close() 788 })) 789 defer ts.Close() 790 791 for _, chunked := range []string{"1", "0"} { 792 c := &Client{Transport: &Transport{}} 793 794 // First fetch something large, but only read some of it. 795 res, err := c.Get(ts.URL + "/?body=large&chunked=" + chunked) 796 if err != nil { 797 t.Fatalf("large get: %v", err) 798 } 799 buf := make([]byte, len(testString)) 800 n, err := io.ReadFull(res.Body, buf) 801 if err != nil { 802 t.Fatalf("partial read of large response: size=%d, %v", n, err) 803 } 804 if e, g := testString, string(buf); e != g { 805 t.Errorf("partial read got %q, expected %q", g, e) 806 } 807 res.Body.Close() 808 // Read on the body, even though it's closed 809 n, err = res.Body.Read(buf) 810 if n != 0 || err == nil { 811 t.Errorf("expected error post-closed large Read; got = %d, %v", n, err) 812 } 813 814 // Then something small. 815 res, err = c.Get(ts.URL + "/?chunked=" + chunked) 816 if err != nil { 817 t.Fatal(err) 818 } 819 body, err := ioutil.ReadAll(res.Body) 820 if err != nil { 821 t.Fatal(err) 822 } 823 if g, e := string(body), testString; g != e { 824 t.Fatalf("body = %q; want %q", g, e) 825 } 826 if g, e := res.Header.Get("Content-Encoding"), ""; g != e { 827 t.Fatalf("Content-Encoding = %q; want %q", g, e) 828 } 829 830 // Read on the body after it's been fully read: 831 n, err = res.Body.Read(buf) 832 if n != 0 || err == nil { 833 t.Errorf("expected Read error after exhausted reads; got %d, %v", n, err) 834 } 835 res.Body.Close() 836 n, err = res.Body.Read(buf) 837 if n != 0 || err == nil { 838 t.Errorf("expected Read error after Close; got %d, %v", n, err) 839 } 840 } 841 842 // And a HEAD request too, because they're always weird. 843 c := &Client{Transport: &Transport{}} 844 res, err := c.Head(ts.URL) 845 if err != nil { 846 t.Fatalf("Head: %v", err) 847 } 848 if res.StatusCode != 200 { 849 t.Errorf("Head status=%d; want=200", res.StatusCode) 850 } 851 } 852 853 // If a request has Expect:100-continue header, the request blocks sending body until the first response. 854 // Premature consumption of the request body should not be occurred. 855 func TestTransportExpect100Continue(t *testing.T) { 856 defer afterTest(t) 857 858 ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { 859 switch req.URL.Path { 860 case "/100": 861 // This endpoint implicitly responds 100 Continue and reads body. 862 if _, err := io.Copy(ioutil.Discard, req.Body); err != nil { 863 t.Error("Failed to read Body", err) 864 } 865 rw.WriteHeader(StatusOK) 866 case "/200": 867 // Go 1.5 adds Connection: close header if the client expect 868 // continue but not entire request body is consumed. 869 rw.WriteHeader(StatusOK) 870 case "/500": 871 rw.WriteHeader(StatusInternalServerError) 872 case "/keepalive": 873 // This hijacked endpoint responds error without Connection:close. 874 _, bufrw, err := rw.(Hijacker).Hijack() 875 if err != nil { 876 log.Fatal(err) 877 } 878 bufrw.WriteString("HTTP/1.1 500 Internal Server Error\r\n") 879 bufrw.WriteString("Content-Length: 0\r\n\r\n") 880 bufrw.Flush() 881 case "/timeout": 882 // This endpoint tries to read body without 100 (Continue) response. 883 // After ExpectContinueTimeout, the reading will be started. 884 conn, bufrw, err := rw.(Hijacker).Hijack() 885 if err != nil { 886 log.Fatal(err) 887 } 888 if _, err := io.CopyN(ioutil.Discard, bufrw, req.ContentLength); err != nil { 889 t.Error("Failed to read Body", err) 890 } 891 bufrw.WriteString("HTTP/1.1 200 OK\r\n\r\n") 892 bufrw.Flush() 893 conn.Close() 894 } 895 896 })) 897 defer ts.Close() 898 899 tests := []struct { 900 path string 901 body []byte 902 sent int 903 status int 904 }{ 905 {path: "/100", body: []byte("hello"), sent: 5, status: 200}, // Got 100 followed by 200, entire body is sent. 906 {path: "/200", body: []byte("hello"), sent: 0, status: 200}, // Got 200 without 100. body isn't sent. 907 {path: "/500", body: []byte("hello"), sent: 0, status: 500}, // Got 500 without 100. body isn't sent. 908 {path: "/keepalive", body: []byte("hello"), sent: 0, status: 500}, // Although without Connection:close, body isn't sent. 909 {path: "/timeout", body: []byte("hello"), sent: 5, status: 200}, // Timeout exceeded and entire body is sent. 910 } 911 912 for i, v := range tests { 913 tr := &Transport{ExpectContinueTimeout: 2 * time.Second} 914 defer tr.CloseIdleConnections() 915 c := &Client{Transport: tr} 916 917 body := bytes.NewReader(v.body) 918 req, err := NewRequest("PUT", ts.URL+v.path, body) 919 if err != nil { 920 t.Fatal(err) 921 } 922 req.Header.Set("Expect", "100-continue") 923 req.ContentLength = int64(len(v.body)) 924 925 resp, err := c.Do(req) 926 if err != nil { 927 t.Fatal(err) 928 } 929 resp.Body.Close() 930 931 sent := len(v.body) - body.Len() 932 if v.status != resp.StatusCode { 933 t.Errorf("test %d: status code should be %d but got %d. (%s)", i, v.status, resp.StatusCode, v.path) 934 } 935 if v.sent != sent { 936 t.Errorf("test %d: sent body should be %d but sent %d. (%s)", i, v.sent, sent, v.path) 937 } 938 } 939 } 940 941 func TestTransportProxy(t *testing.T) { 942 defer afterTest(t) 943 ch := make(chan string, 1) 944 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 945 ch <- "real server" 946 })) 947 defer ts.Close() 948 proxy := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 949 ch <- "proxy for " + r.URL.String() 950 })) 951 defer proxy.Close() 952 953 pu, err := url.Parse(proxy.URL) 954 if err != nil { 955 t.Fatal(err) 956 } 957 c := &Client{Transport: &Transport{Proxy: ProxyURL(pu)}} 958 c.Head(ts.URL) 959 got := <-ch 960 want := "proxy for " + ts.URL + "/" 961 if got != want { 962 t.Errorf("want %q, got %q", want, got) 963 } 964 } 965 966 // Issue 16997: test transport dial preserves typed errors 967 func TestTransportDialPreservesNetOpProxyError(t *testing.T) { 968 defer afterTest(t) 969 970 var errDial = errors.New("some dial error") 971 972 tr := &Transport{ 973 Proxy: func(*Request) (*url.URL, error) { 974 return url.Parse("http://proxy.fake.tld/") 975 }, 976 Dial: func(string, string) (net.Conn, error) { 977 return nil, errDial 978 }, 979 } 980 defer tr.CloseIdleConnections() 981 982 c := &Client{Transport: tr} 983 req, _ := NewRequest("GET", "http://fake.tld", nil) 984 res, err := c.Do(req) 985 if err == nil { 986 res.Body.Close() 987 t.Fatal("wanted a non-nil error") 988 } 989 990 uerr, ok := err.(*url.Error) 991 if !ok { 992 t.Fatalf("got %T, want *url.Error", err) 993 } 994 oe, ok := uerr.Err.(*net.OpError) 995 if !ok { 996 t.Fatalf("url.Error.Err = %T; want *net.OpError", uerr.Err) 997 } 998 want := &net.OpError{ 999 Op: "proxyconnect", 1000 Net: "tcp", 1001 Err: errDial, // original error, unwrapped. 1002 } 1003 if !reflect.DeepEqual(oe, want) { 1004 t.Errorf("Got error %#v; want %#v", oe, want) 1005 } 1006 } 1007 1008 // TestTransportGzipRecursive sends a gzip quine and checks that the 1009 // client gets the same value back. This is more cute than anything, 1010 // but checks that we don't recurse forever, and checks that 1011 // Content-Encoding is removed. 1012 func TestTransportGzipRecursive(t *testing.T) { 1013 defer afterTest(t) 1014 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1015 w.Header().Set("Content-Encoding", "gzip") 1016 w.Write(rgz) 1017 })) 1018 defer ts.Close() 1019 1020 tr := &Transport{} 1021 defer tr.CloseIdleConnections() 1022 c := &Client{Transport: tr} 1023 res, err := c.Get(ts.URL) 1024 if err != nil { 1025 t.Fatal(err) 1026 } 1027 body, err := ioutil.ReadAll(res.Body) 1028 if err != nil { 1029 t.Fatal(err) 1030 } 1031 if !bytes.Equal(body, rgz) { 1032 t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x", 1033 body, rgz) 1034 } 1035 if g, e := res.Header.Get("Content-Encoding"), ""; g != e { 1036 t.Fatalf("Content-Encoding = %q; want %q", g, e) 1037 } 1038 } 1039 1040 // golang.org/issue/7750: request fails when server replies with 1041 // a short gzip body 1042 func TestTransportGzipShort(t *testing.T) { 1043 defer afterTest(t) 1044 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1045 w.Header().Set("Content-Encoding", "gzip") 1046 w.Write([]byte{0x1f, 0x8b}) 1047 })) 1048 defer ts.Close() 1049 1050 tr := &Transport{} 1051 defer tr.CloseIdleConnections() 1052 c := &Client{Transport: tr} 1053 res, err := c.Get(ts.URL) 1054 if err != nil { 1055 t.Fatal(err) 1056 } 1057 defer res.Body.Close() 1058 _, err = ioutil.ReadAll(res.Body) 1059 if err == nil { 1060 t.Fatal("Expect an error from reading a body.") 1061 } 1062 if err != io.ErrUnexpectedEOF { 1063 t.Errorf("ReadAll error = %v; want io.ErrUnexpectedEOF", err) 1064 } 1065 } 1066 1067 // Wait until number of goroutines is no greater than nmax, or time out. 1068 func waitNumGoroutine(nmax int) int { 1069 nfinal := runtime.NumGoroutine() 1070 for ntries := 10; ntries > 0 && nfinal > nmax; ntries-- { 1071 time.Sleep(50 * time.Millisecond) 1072 runtime.GC() 1073 nfinal = runtime.NumGoroutine() 1074 } 1075 return nfinal 1076 } 1077 1078 // tests that persistent goroutine connections shut down when no longer desired. 1079 func TestTransportPersistConnLeak(t *testing.T) { 1080 setParallel(t) 1081 defer afterTest(t) 1082 gotReqCh := make(chan bool) 1083 unblockCh := make(chan bool) 1084 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1085 gotReqCh <- true 1086 <-unblockCh 1087 w.Header().Set("Content-Length", "0") 1088 w.WriteHeader(204) 1089 })) 1090 defer ts.Close() 1091 1092 tr := &Transport{} 1093 c := &Client{Transport: tr} 1094 1095 n0 := runtime.NumGoroutine() 1096 1097 const numReq = 25 1098 didReqCh := make(chan bool) 1099 for i := 0; i < numReq; i++ { 1100 go func() { 1101 res, err := c.Get(ts.URL) 1102 didReqCh <- true 1103 if err != nil { 1104 t.Errorf("client fetch error: %v", err) 1105 return 1106 } 1107 res.Body.Close() 1108 }() 1109 } 1110 1111 // Wait for all goroutines to be stuck in the Handler. 1112 for i := 0; i < numReq; i++ { 1113 <-gotReqCh 1114 } 1115 1116 nhigh := runtime.NumGoroutine() 1117 1118 // Tell all handlers to unblock and reply. 1119 for i := 0; i < numReq; i++ { 1120 unblockCh <- true 1121 } 1122 1123 // Wait for all HTTP clients to be done. 1124 for i := 0; i < numReq; i++ { 1125 <-didReqCh 1126 } 1127 1128 tr.CloseIdleConnections() 1129 nfinal := waitNumGoroutine(n0 + 5) 1130 1131 growth := nfinal - n0 1132 1133 // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. 1134 // Previously we were leaking one per numReq. 1135 if int(growth) > 5 { 1136 t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) 1137 t.Error("too many new goroutines") 1138 } 1139 } 1140 1141 // golang.org/issue/4531: Transport leaks goroutines when 1142 // request.ContentLength is explicitly short 1143 func TestTransportPersistConnLeakShortBody(t *testing.T) { 1144 setParallel(t) 1145 defer afterTest(t) 1146 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1147 })) 1148 defer ts.Close() 1149 1150 tr := &Transport{} 1151 c := &Client{Transport: tr} 1152 1153 n0 := runtime.NumGoroutine() 1154 body := []byte("Hello") 1155 for i := 0; i < 20; i++ { 1156 req, err := NewRequest("POST", ts.URL, bytes.NewReader(body)) 1157 if err != nil { 1158 t.Fatal(err) 1159 } 1160 req.ContentLength = int64(len(body) - 2) // explicitly short 1161 _, err = c.Do(req) 1162 if err == nil { 1163 t.Fatal("Expect an error from writing too long of a body.") 1164 } 1165 } 1166 nhigh := runtime.NumGoroutine() 1167 tr.CloseIdleConnections() 1168 nfinal := waitNumGoroutine(n0 + 5) 1169 1170 growth := nfinal - n0 1171 1172 // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. 1173 // Previously we were leaking one per numReq. 1174 t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) 1175 if int(growth) > 5 { 1176 t.Error("too many new goroutines") 1177 } 1178 } 1179 1180 // This used to crash; https://golang.org/issue/3266 1181 func TestTransportIdleConnCrash(t *testing.T) { 1182 defer afterTest(t) 1183 tr := &Transport{} 1184 c := &Client{Transport: tr} 1185 1186 unblockCh := make(chan bool, 1) 1187 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1188 <-unblockCh 1189 tr.CloseIdleConnections() 1190 })) 1191 defer ts.Close() 1192 1193 didreq := make(chan bool) 1194 go func() { 1195 res, err := c.Get(ts.URL) 1196 if err != nil { 1197 t.Error(err) 1198 } else { 1199 res.Body.Close() // returns idle conn 1200 } 1201 didreq <- true 1202 }() 1203 unblockCh <- true 1204 <-didreq 1205 } 1206 1207 // Test that the transport doesn't close the TCP connection early, 1208 // before the response body has been read. This was a regression 1209 // which sadly lacked a triggering test. The large response body made 1210 // the old race easier to trigger. 1211 func TestIssue3644(t *testing.T) { 1212 defer afterTest(t) 1213 const numFoos = 5000 1214 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1215 w.Header().Set("Connection", "close") 1216 for i := 0; i < numFoos; i++ { 1217 w.Write([]byte("foo ")) 1218 } 1219 })) 1220 defer ts.Close() 1221 tr := &Transport{} 1222 c := &Client{Transport: tr} 1223 res, err := c.Get(ts.URL) 1224 if err != nil { 1225 t.Fatal(err) 1226 } 1227 defer res.Body.Close() 1228 bs, err := ioutil.ReadAll(res.Body) 1229 if err != nil { 1230 t.Fatal(err) 1231 } 1232 if len(bs) != numFoos*len("foo ") { 1233 t.Errorf("unexpected response length") 1234 } 1235 } 1236 1237 // Test that a client receives a server's reply, even if the server doesn't read 1238 // the entire request body. 1239 func TestIssue3595(t *testing.T) { 1240 defer afterTest(t) 1241 const deniedMsg = "sorry, denied." 1242 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1243 Error(w, deniedMsg, StatusUnauthorized) 1244 })) 1245 defer ts.Close() 1246 tr := &Transport{} 1247 c := &Client{Transport: tr} 1248 res, err := c.Post(ts.URL, "application/octet-stream", neverEnding('a')) 1249 if err != nil { 1250 t.Errorf("Post: %v", err) 1251 return 1252 } 1253 got, err := ioutil.ReadAll(res.Body) 1254 if err != nil { 1255 t.Fatalf("Body ReadAll: %v", err) 1256 } 1257 if !strings.Contains(string(got), deniedMsg) { 1258 t.Errorf("Known bug: response %q does not contain %q", got, deniedMsg) 1259 } 1260 } 1261 1262 // From https://golang.org/issue/4454 , 1263 // "client fails to handle requests with no body and chunked encoding" 1264 func TestChunkedNoContent(t *testing.T) { 1265 defer afterTest(t) 1266 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1267 w.WriteHeader(StatusNoContent) 1268 })) 1269 defer ts.Close() 1270 1271 for _, closeBody := range []bool{true, false} { 1272 c := &Client{Transport: &Transport{}} 1273 const n = 4 1274 for i := 1; i <= n; i++ { 1275 res, err := c.Get(ts.URL) 1276 if err != nil { 1277 t.Errorf("closingBody=%v, req %d/%d: %v", closeBody, i, n, err) 1278 } else { 1279 if closeBody { 1280 res.Body.Close() 1281 } 1282 } 1283 } 1284 } 1285 } 1286 1287 func TestTransportConcurrency(t *testing.T) { 1288 defer afterTest(t) 1289 maxProcs, numReqs := 16, 500 1290 if testing.Short() { 1291 maxProcs, numReqs = 4, 50 1292 } 1293 defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(maxProcs)) 1294 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1295 fmt.Fprintf(w, "%v", r.FormValue("echo")) 1296 })) 1297 defer ts.Close() 1298 1299 var wg sync.WaitGroup 1300 wg.Add(numReqs) 1301 1302 // Due to the Transport's "socket late binding" (see 1303 // idleConnCh in transport.go), the numReqs HTTP requests 1304 // below can finish with a dial still outstanding. To keep 1305 // the leak checker happy, keep track of pending dials and 1306 // wait for them to finish (and be closed or returned to the 1307 // idle pool) before we close idle connections. 1308 SetPendingDialHooks(func() { wg.Add(1) }, wg.Done) 1309 defer SetPendingDialHooks(nil, nil) 1310 1311 tr := &Transport{} 1312 defer tr.CloseIdleConnections() 1313 1314 c := &Client{Transport: tr} 1315 reqs := make(chan string) 1316 defer close(reqs) 1317 1318 for i := 0; i < maxProcs*2; i++ { 1319 go func() { 1320 for req := range reqs { 1321 res, err := c.Get(ts.URL + "/?echo=" + req) 1322 if err != nil { 1323 t.Errorf("error on req %s: %v", req, err) 1324 wg.Done() 1325 continue 1326 } 1327 all, err := ioutil.ReadAll(res.Body) 1328 if err != nil { 1329 t.Errorf("read error on req %s: %v", req, err) 1330 wg.Done() 1331 continue 1332 } 1333 if string(all) != req { 1334 t.Errorf("body of req %s = %q; want %q", req, all, req) 1335 } 1336 res.Body.Close() 1337 wg.Done() 1338 } 1339 }() 1340 } 1341 for i := 0; i < numReqs; i++ { 1342 reqs <- fmt.Sprintf("request-%d", i) 1343 } 1344 wg.Wait() 1345 } 1346 1347 func TestIssue4191_InfiniteGetTimeout(t *testing.T) { 1348 if runtime.GOOS == "plan9" { 1349 t.Skip("skipping test; see https://golang.org/issue/7237") 1350 } 1351 defer afterTest(t) 1352 const debug = false 1353 mux := NewServeMux() 1354 mux.HandleFunc("/get", func(w ResponseWriter, r *Request) { 1355 io.Copy(w, neverEnding('a')) 1356 }) 1357 ts := httptest.NewServer(mux) 1358 timeout := 100 * time.Millisecond 1359 1360 client := &Client{ 1361 Transport: &Transport{ 1362 Dial: func(n, addr string) (net.Conn, error) { 1363 conn, err := net.Dial(n, addr) 1364 if err != nil { 1365 return nil, err 1366 } 1367 conn.SetDeadline(time.Now().Add(timeout)) 1368 if debug { 1369 conn = NewLoggingConn("client", conn) 1370 } 1371 return conn, nil 1372 }, 1373 DisableKeepAlives: true, 1374 }, 1375 } 1376 1377 getFailed := false 1378 nRuns := 5 1379 if testing.Short() { 1380 nRuns = 1 1381 } 1382 for i := 0; i < nRuns; i++ { 1383 if debug { 1384 println("run", i+1, "of", nRuns) 1385 } 1386 sres, err := client.Get(ts.URL + "/get") 1387 if err != nil { 1388 if !getFailed { 1389 // Make the timeout longer, once. 1390 getFailed = true 1391 t.Logf("increasing timeout") 1392 i-- 1393 timeout *= 10 1394 continue 1395 } 1396 t.Errorf("Error issuing GET: %v", err) 1397 break 1398 } 1399 _, err = io.Copy(ioutil.Discard, sres.Body) 1400 if err == nil { 1401 t.Errorf("Unexpected successful copy") 1402 break 1403 } 1404 } 1405 if debug { 1406 println("tests complete; waiting for handlers to finish") 1407 } 1408 ts.Close() 1409 } 1410 1411 func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { 1412 if runtime.GOOS == "plan9" { 1413 t.Skip("skipping test; see https://golang.org/issue/7237") 1414 } 1415 defer afterTest(t) 1416 const debug = false 1417 mux := NewServeMux() 1418 mux.HandleFunc("/get", func(w ResponseWriter, r *Request) { 1419 io.Copy(w, neverEnding('a')) 1420 }) 1421 mux.HandleFunc("/put", func(w ResponseWriter, r *Request) { 1422 defer r.Body.Close() 1423 io.Copy(ioutil.Discard, r.Body) 1424 }) 1425 ts := httptest.NewServer(mux) 1426 timeout := 100 * time.Millisecond 1427 1428 client := &Client{ 1429 Transport: &Transport{ 1430 Dial: func(n, addr string) (net.Conn, error) { 1431 conn, err := net.Dial(n, addr) 1432 if err != nil { 1433 return nil, err 1434 } 1435 conn.SetDeadline(time.Now().Add(timeout)) 1436 if debug { 1437 conn = NewLoggingConn("client", conn) 1438 } 1439 return conn, nil 1440 }, 1441 DisableKeepAlives: true, 1442 }, 1443 } 1444 1445 getFailed := false 1446 nRuns := 5 1447 if testing.Short() { 1448 nRuns = 1 1449 } 1450 for i := 0; i < nRuns; i++ { 1451 if debug { 1452 println("run", i+1, "of", nRuns) 1453 } 1454 sres, err := client.Get(ts.URL + "/get") 1455 if err != nil { 1456 if !getFailed { 1457 // Make the timeout longer, once. 1458 getFailed = true 1459 t.Logf("increasing timeout") 1460 i-- 1461 timeout *= 10 1462 continue 1463 } 1464 t.Errorf("Error issuing GET: %v", err) 1465 break 1466 } 1467 req, _ := NewRequest("PUT", ts.URL+"/put", sres.Body) 1468 _, err = client.Do(req) 1469 if err == nil { 1470 sres.Body.Close() 1471 t.Errorf("Unexpected successful PUT") 1472 break 1473 } 1474 sres.Body.Close() 1475 } 1476 if debug { 1477 println("tests complete; waiting for handlers to finish") 1478 } 1479 ts.Close() 1480 } 1481 1482 func TestTransportResponseHeaderTimeout(t *testing.T) { 1483 setParallel(t) 1484 defer afterTest(t) 1485 if testing.Short() { 1486 t.Skip("skipping timeout test in -short mode") 1487 } 1488 inHandler := make(chan bool, 1) 1489 mux := NewServeMux() 1490 mux.HandleFunc("/fast", func(w ResponseWriter, r *Request) { 1491 inHandler <- true 1492 }) 1493 mux.HandleFunc("/slow", func(w ResponseWriter, r *Request) { 1494 inHandler <- true 1495 time.Sleep(2 * time.Second) 1496 }) 1497 ts := httptest.NewServer(mux) 1498 defer ts.Close() 1499 1500 tr := &Transport{ 1501 ResponseHeaderTimeout: 500 * time.Millisecond, 1502 } 1503 defer tr.CloseIdleConnections() 1504 c := &Client{Transport: tr} 1505 1506 tests := []struct { 1507 path string 1508 want int 1509 wantErr string 1510 }{ 1511 {path: "/fast", want: 200}, 1512 {path: "/slow", wantErr: "timeout awaiting response headers"}, 1513 {path: "/fast", want: 200}, 1514 } 1515 for i, tt := range tests { 1516 res, err := c.Get(ts.URL + tt.path) 1517 select { 1518 case <-inHandler: 1519 case <-time.After(5 * time.Second): 1520 t.Errorf("never entered handler for test index %d, %s", i, tt.path) 1521 continue 1522 } 1523 if err != nil { 1524 uerr, ok := err.(*url.Error) 1525 if !ok { 1526 t.Errorf("error is not an url.Error; got: %#v", err) 1527 continue 1528 } 1529 nerr, ok := uerr.Err.(net.Error) 1530 if !ok { 1531 t.Errorf("error does not satisfy net.Error interface; got: %#v", err) 1532 continue 1533 } 1534 if !nerr.Timeout() { 1535 t.Errorf("want timeout error; got: %q", nerr) 1536 continue 1537 } 1538 if strings.Contains(err.Error(), tt.wantErr) { 1539 continue 1540 } 1541 t.Errorf("%d. unexpected error: %v", i, err) 1542 continue 1543 } 1544 if tt.wantErr != "" { 1545 t.Errorf("%d. no error. expected error: %v", i, tt.wantErr) 1546 continue 1547 } 1548 if res.StatusCode != tt.want { 1549 t.Errorf("%d for path %q status = %d; want %d", i, tt.path, res.StatusCode, tt.want) 1550 } 1551 } 1552 } 1553 1554 func TestTransportCancelRequest(t *testing.T) { 1555 setParallel(t) 1556 defer afterTest(t) 1557 if testing.Short() { 1558 t.Skip("skipping test in -short mode") 1559 } 1560 unblockc := make(chan bool) 1561 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1562 fmt.Fprintf(w, "Hello") 1563 w.(Flusher).Flush() // send headers and some body 1564 <-unblockc 1565 })) 1566 defer ts.Close() 1567 defer close(unblockc) 1568 1569 tr := &Transport{} 1570 defer tr.CloseIdleConnections() 1571 c := &Client{Transport: tr} 1572 1573 req, _ := NewRequest("GET", ts.URL, nil) 1574 res, err := c.Do(req) 1575 if err != nil { 1576 t.Fatal(err) 1577 } 1578 go func() { 1579 time.Sleep(1 * time.Second) 1580 tr.CancelRequest(req) 1581 }() 1582 t0 := time.Now() 1583 body, err := ioutil.ReadAll(res.Body) 1584 d := time.Since(t0) 1585 1586 if err != ExportErrRequestCanceled { 1587 t.Errorf("Body.Read error = %v; want errRequestCanceled", err) 1588 } 1589 if string(body) != "Hello" { 1590 t.Errorf("Body = %q; want Hello", body) 1591 } 1592 if d < 500*time.Millisecond { 1593 t.Errorf("expected ~1 second delay; got %v", d) 1594 } 1595 // Verify no outstanding requests after readLoop/writeLoop 1596 // goroutines shut down. 1597 for tries := 5; tries > 0; tries-- { 1598 n := tr.NumPendingRequestsForTesting() 1599 if n == 0 { 1600 break 1601 } 1602 time.Sleep(100 * time.Millisecond) 1603 if tries == 1 { 1604 t.Errorf("pending requests = %d; want 0", n) 1605 } 1606 } 1607 } 1608 1609 func TestTransportCancelRequestInDial(t *testing.T) { 1610 defer afterTest(t) 1611 if testing.Short() { 1612 t.Skip("skipping test in -short mode") 1613 } 1614 var logbuf bytes.Buffer 1615 eventLog := log.New(&logbuf, "", 0) 1616 1617 unblockDial := make(chan bool) 1618 defer close(unblockDial) 1619 1620 inDial := make(chan bool) 1621 tr := &Transport{ 1622 Dial: func(network, addr string) (net.Conn, error) { 1623 eventLog.Println("dial: blocking") 1624 inDial <- true 1625 <-unblockDial 1626 return nil, errors.New("nope") 1627 }, 1628 } 1629 cl := &Client{Transport: tr} 1630 gotres := make(chan bool) 1631 req, _ := NewRequest("GET", "http://something.no-network.tld/", nil) 1632 go func() { 1633 _, err := cl.Do(req) 1634 eventLog.Printf("Get = %v", err) 1635 gotres <- true 1636 }() 1637 1638 select { 1639 case <-inDial: 1640 case <-time.After(5 * time.Second): 1641 t.Fatal("timeout; never saw blocking dial") 1642 } 1643 1644 eventLog.Printf("canceling") 1645 tr.CancelRequest(req) 1646 tr.CancelRequest(req) // used to panic on second call 1647 1648 select { 1649 case <-gotres: 1650 case <-time.After(5 * time.Second): 1651 panic("hang. events are: " + logbuf.String()) 1652 } 1653 1654 got := logbuf.String() 1655 want := `dial: blocking 1656 canceling 1657 Get = Get http://something.no-network.tld/: net/http: request canceled while waiting for connection 1658 ` 1659 if got != want { 1660 t.Errorf("Got events:\n%s\nWant:\n%s", got, want) 1661 } 1662 } 1663 1664 func TestCancelRequestWithChannel(t *testing.T) { 1665 setParallel(t) 1666 defer afterTest(t) 1667 if testing.Short() { 1668 t.Skip("skipping test in -short mode") 1669 } 1670 unblockc := make(chan bool) 1671 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1672 fmt.Fprintf(w, "Hello") 1673 w.(Flusher).Flush() // send headers and some body 1674 <-unblockc 1675 })) 1676 defer ts.Close() 1677 defer close(unblockc) 1678 1679 tr := &Transport{} 1680 defer tr.CloseIdleConnections() 1681 c := &Client{Transport: tr} 1682 1683 req, _ := NewRequest("GET", ts.URL, nil) 1684 ch := make(chan struct{}) 1685 req.Cancel = ch 1686 1687 res, err := c.Do(req) 1688 if err != nil { 1689 t.Fatal(err) 1690 } 1691 go func() { 1692 time.Sleep(1 * time.Second) 1693 close(ch) 1694 }() 1695 t0 := time.Now() 1696 body, err := ioutil.ReadAll(res.Body) 1697 d := time.Since(t0) 1698 1699 if err != ExportErrRequestCanceled { 1700 t.Errorf("Body.Read error = %v; want errRequestCanceled", err) 1701 } 1702 if string(body) != "Hello" { 1703 t.Errorf("Body = %q; want Hello", body) 1704 } 1705 if d < 500*time.Millisecond { 1706 t.Errorf("expected ~1 second delay; got %v", d) 1707 } 1708 // Verify no outstanding requests after readLoop/writeLoop 1709 // goroutines shut down. 1710 for tries := 5; tries > 0; tries-- { 1711 n := tr.NumPendingRequestsForTesting() 1712 if n == 0 { 1713 break 1714 } 1715 time.Sleep(100 * time.Millisecond) 1716 if tries == 1 { 1717 t.Errorf("pending requests = %d; want 0", n) 1718 } 1719 } 1720 } 1721 1722 func TestCancelRequestWithChannelBeforeDo_Cancel(t *testing.T) { 1723 testCancelRequestWithChannelBeforeDo(t, false) 1724 } 1725 func TestCancelRequestWithChannelBeforeDo_Context(t *testing.T) { 1726 testCancelRequestWithChannelBeforeDo(t, true) 1727 } 1728 func testCancelRequestWithChannelBeforeDo(t *testing.T, withCtx bool) { 1729 setParallel(t) 1730 defer afterTest(t) 1731 unblockc := make(chan bool) 1732 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1733 <-unblockc 1734 })) 1735 defer ts.Close() 1736 defer close(unblockc) 1737 1738 tr := &Transport{} 1739 defer tr.CloseIdleConnections() 1740 c := &Client{Transport: tr} 1741 1742 req, _ := NewRequest("GET", ts.URL, nil) 1743 if withCtx { 1744 ctx, cancel := context.WithCancel(context.Background()) 1745 cancel() 1746 req = req.WithContext(ctx) 1747 } else { 1748 ch := make(chan struct{}) 1749 req.Cancel = ch 1750 close(ch) 1751 } 1752 1753 _, err := c.Do(req) 1754 if ue, ok := err.(*url.Error); ok { 1755 err = ue.Err 1756 } 1757 if withCtx { 1758 if err != context.Canceled { 1759 t.Errorf("Do error = %v; want %v", err, context.Canceled) 1760 } 1761 } else { 1762 if err == nil || !strings.Contains(err.Error(), "canceled") { 1763 t.Errorf("Do error = %v; want cancelation", err) 1764 } 1765 } 1766 } 1767 1768 // Issue 11020. The returned error message should be errRequestCanceled 1769 func TestTransportCancelBeforeResponseHeaders(t *testing.T) { 1770 defer afterTest(t) 1771 1772 serverConnCh := make(chan net.Conn, 1) 1773 tr := &Transport{ 1774 Dial: func(network, addr string) (net.Conn, error) { 1775 cc, sc := net.Pipe() 1776 serverConnCh <- sc 1777 return cc, nil 1778 }, 1779 } 1780 defer tr.CloseIdleConnections() 1781 errc := make(chan error, 1) 1782 req, _ := NewRequest("GET", "http://example.com/", nil) 1783 go func() { 1784 _, err := tr.RoundTrip(req) 1785 errc <- err 1786 }() 1787 1788 sc := <-serverConnCh 1789 verb := make([]byte, 3) 1790 if _, err := io.ReadFull(sc, verb); err != nil { 1791 t.Errorf("Error reading HTTP verb from server: %v", err) 1792 } 1793 if string(verb) != "GET" { 1794 t.Errorf("server received %q; want GET", verb) 1795 } 1796 defer sc.Close() 1797 1798 tr.CancelRequest(req) 1799 1800 err := <-errc 1801 if err == nil { 1802 t.Fatalf("unexpected success from RoundTrip") 1803 } 1804 if err != ExportErrRequestCanceled { 1805 t.Errorf("RoundTrip error = %v; want ExportErrRequestCanceled", err) 1806 } 1807 } 1808 1809 // golang.org/issue/3672 -- Client can't close HTTP stream 1810 // Calling Close on a Response.Body used to just read until EOF. 1811 // Now it actually closes the TCP connection. 1812 func TestTransportCloseResponseBody(t *testing.T) { 1813 defer afterTest(t) 1814 writeErr := make(chan error, 1) 1815 msg := []byte("young\n") 1816 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 1817 for { 1818 _, err := w.Write(msg) 1819 if err != nil { 1820 writeErr <- err 1821 return 1822 } 1823 w.(Flusher).Flush() 1824 } 1825 })) 1826 defer ts.Close() 1827 1828 tr := &Transport{} 1829 defer tr.CloseIdleConnections() 1830 c := &Client{Transport: tr} 1831 1832 req, _ := NewRequest("GET", ts.URL, nil) 1833 defer tr.CancelRequest(req) 1834 1835 res, err := c.Do(req) 1836 if err != nil { 1837 t.Fatal(err) 1838 } 1839 1840 const repeats = 3 1841 buf := make([]byte, len(msg)*repeats) 1842 want := bytes.Repeat(msg, repeats) 1843 1844 _, err = io.ReadFull(res.Body, buf) 1845 if err != nil { 1846 t.Fatal(err) 1847 } 1848 if !bytes.Equal(buf, want) { 1849 t.Fatalf("read %q; want %q", buf, want) 1850 } 1851 didClose := make(chan error, 1) 1852 go func() { 1853 didClose <- res.Body.Close() 1854 }() 1855 select { 1856 case err := <-didClose: 1857 if err != nil { 1858 t.Errorf("Close = %v", err) 1859 } 1860 case <-time.After(10 * time.Second): 1861 t.Fatal("too long waiting for close") 1862 } 1863 select { 1864 case err := <-writeErr: 1865 if err == nil { 1866 t.Errorf("expected non-nil write error") 1867 } 1868 case <-time.After(10 * time.Second): 1869 t.Fatal("too long waiting for write error") 1870 } 1871 } 1872 1873 type fooProto struct{} 1874 1875 func (fooProto) RoundTrip(req *Request) (*Response, error) { 1876 res := &Response{ 1877 Status: "200 OK", 1878 StatusCode: 200, 1879 Header: make(Header), 1880 Body: ioutil.NopCloser(strings.NewReader("You wanted " + req.URL.String())), 1881 } 1882 return res, nil 1883 } 1884 1885 func TestTransportAltProto(t *testing.T) { 1886 defer afterTest(t) 1887 tr := &Transport{} 1888 c := &Client{Transport: tr} 1889 tr.RegisterProtocol("foo", fooProto{}) 1890 res, err := c.Get("foo://bar.com/path") 1891 if err != nil { 1892 t.Fatal(err) 1893 } 1894 bodyb, err := ioutil.ReadAll(res.Body) 1895 if err != nil { 1896 t.Fatal(err) 1897 } 1898 body := string(bodyb) 1899 if e := "You wanted foo://bar.com/path"; body != e { 1900 t.Errorf("got response %q, want %q", body, e) 1901 } 1902 } 1903 1904 func TestTransportNoHost(t *testing.T) { 1905 defer afterTest(t) 1906 tr := &Transport{} 1907 _, err := tr.RoundTrip(&Request{ 1908 Header: make(Header), 1909 URL: &url.URL{ 1910 Scheme: "http", 1911 }, 1912 }) 1913 want := "http: no Host in request URL" 1914 if got := fmt.Sprint(err); got != want { 1915 t.Errorf("error = %v; want %q", err, want) 1916 } 1917 } 1918 1919 // Issue 13311 1920 func TestTransportEmptyMethod(t *testing.T) { 1921 req, _ := NewRequest("GET", "http://foo.com/", nil) 1922 req.Method = "" // docs say "For client requests an empty string means GET" 1923 got, err := httputil.DumpRequestOut(req, false) // DumpRequestOut uses Transport 1924 if err != nil { 1925 t.Fatal(err) 1926 } 1927 if !strings.Contains(string(got), "GET ") { 1928 t.Fatalf("expected substring 'GET '; got: %s", got) 1929 } 1930 } 1931 1932 func TestTransportSocketLateBinding(t *testing.T) { 1933 defer afterTest(t) 1934 1935 mux := NewServeMux() 1936 fooGate := make(chan bool, 1) 1937 mux.HandleFunc("/foo", func(w ResponseWriter, r *Request) { 1938 w.Header().Set("foo-ipport", r.RemoteAddr) 1939 w.(Flusher).Flush() 1940 <-fooGate 1941 }) 1942 mux.HandleFunc("/bar", func(w ResponseWriter, r *Request) { 1943 w.Header().Set("bar-ipport", r.RemoteAddr) 1944 }) 1945 ts := httptest.NewServer(mux) 1946 defer ts.Close() 1947 1948 dialGate := make(chan bool, 1) 1949 tr := &Transport{ 1950 Dial: func(n, addr string) (net.Conn, error) { 1951 if <-dialGate { 1952 return net.Dial(n, addr) 1953 } 1954 return nil, errors.New("manually closed") 1955 }, 1956 DisableKeepAlives: false, 1957 } 1958 defer tr.CloseIdleConnections() 1959 c := &Client{ 1960 Transport: tr, 1961 } 1962 1963 dialGate <- true // only allow one dial 1964 fooRes, err := c.Get(ts.URL + "/foo") 1965 if err != nil { 1966 t.Fatal(err) 1967 } 1968 fooAddr := fooRes.Header.Get("foo-ipport") 1969 if fooAddr == "" { 1970 t.Fatal("No addr on /foo request") 1971 } 1972 time.AfterFunc(200*time.Millisecond, func() { 1973 // let the foo response finish so we can use its 1974 // connection for /bar 1975 fooGate <- true 1976 io.Copy(ioutil.Discard, fooRes.Body) 1977 fooRes.Body.Close() 1978 }) 1979 1980 barRes, err := c.Get(ts.URL + "/bar") 1981 if err != nil { 1982 t.Fatal(err) 1983 } 1984 barAddr := barRes.Header.Get("bar-ipport") 1985 if barAddr != fooAddr { 1986 t.Fatalf("/foo came from conn %q; /bar came from %q instead", fooAddr, barAddr) 1987 } 1988 barRes.Body.Close() 1989 dialGate <- false 1990 } 1991 1992 // Issue 2184 1993 func TestTransportReading100Continue(t *testing.T) { 1994 defer afterTest(t) 1995 1996 const numReqs = 5 1997 reqBody := func(n int) string { return fmt.Sprintf("request body %d", n) } 1998 reqID := func(n int) string { return fmt.Sprintf("REQ-ID-%d", n) } 1999 2000 send100Response := func(w *io.PipeWriter, r *io.PipeReader) { 2001 defer w.Close() 2002 defer r.Close() 2003 br := bufio.NewReader(r) 2004 n := 0 2005 for { 2006 n++ 2007 req, err := ReadRequest(br) 2008 if err == io.EOF { 2009 return 2010 } 2011 if err != nil { 2012 t.Error(err) 2013 return 2014 } 2015 slurp, err := ioutil.ReadAll(req.Body) 2016 if err != nil { 2017 t.Errorf("Server request body slurp: %v", err) 2018 return 2019 } 2020 id := req.Header.Get("Request-Id") 2021 resCode := req.Header.Get("X-Want-Response-Code") 2022 if resCode == "" { 2023 resCode = "100 Continue" 2024 if string(slurp) != reqBody(n) { 2025 t.Errorf("Server got %q, %v; want %q", slurp, err, reqBody(n)) 2026 } 2027 } 2028 body := fmt.Sprintf("Response number %d", n) 2029 v := []byte(strings.Replace(fmt.Sprintf(`HTTP/1.1 %s 2030 Date: Thu, 28 Feb 2013 17:55:41 GMT 2031 2032 HTTP/1.1 200 OK 2033 Content-Type: text/html 2034 Echo-Request-Id: %s 2035 Content-Length: %d 2036 2037 %s`, resCode, id, len(body), body), "\n", "\r\n", -1)) 2038 w.Write(v) 2039 if id == reqID(numReqs) { 2040 return 2041 } 2042 } 2043 2044 } 2045 2046 tr := &Transport{ 2047 Dial: func(n, addr string) (net.Conn, error) { 2048 sr, sw := io.Pipe() // server read/write 2049 cr, cw := io.Pipe() // client read/write 2050 conn := &rwTestConn{ 2051 Reader: cr, 2052 Writer: sw, 2053 closeFunc: func() error { 2054 sw.Close() 2055 cw.Close() 2056 return nil 2057 }, 2058 } 2059 go send100Response(cw, sr) 2060 return conn, nil 2061 }, 2062 DisableKeepAlives: false, 2063 } 2064 defer tr.CloseIdleConnections() 2065 c := &Client{Transport: tr} 2066 2067 testResponse := func(req *Request, name string, wantCode int) { 2068 res, err := c.Do(req) 2069 if err != nil { 2070 t.Fatalf("%s: Do: %v", name, err) 2071 } 2072 if res.StatusCode != wantCode { 2073 t.Fatalf("%s: Response Statuscode=%d; want %d", name, res.StatusCode, wantCode) 2074 } 2075 if id, idBack := req.Header.Get("Request-Id"), res.Header.Get("Echo-Request-Id"); id != "" && id != idBack { 2076 t.Errorf("%s: response id %q != request id %q", name, idBack, id) 2077 } 2078 _, err = ioutil.ReadAll(res.Body) 2079 if err != nil { 2080 t.Fatalf("%s: Slurp error: %v", name, err) 2081 } 2082 } 2083 2084 // Few 100 responses, making sure we're not off-by-one. 2085 for i := 1; i <= numReqs; i++ { 2086 req, _ := NewRequest("POST", "http://dummy.tld/", strings.NewReader(reqBody(i))) 2087 req.Header.Set("Request-Id", reqID(i)) 2088 testResponse(req, fmt.Sprintf("100, %d/%d", i, numReqs), 200) 2089 } 2090 2091 // And some other informational 1xx but non-100 responses, to test 2092 // we return them but don't re-use the connection. 2093 for i := 1; i <= numReqs; i++ { 2094 req, _ := NewRequest("POST", "http://other.tld/", strings.NewReader(reqBody(i))) 2095 req.Header.Set("X-Want-Response-Code", "123 Sesame Street") 2096 testResponse(req, fmt.Sprintf("123, %d/%d", i, numReqs), 123) 2097 } 2098 } 2099 2100 type proxyFromEnvTest struct { 2101 req string // URL to fetch; blank means "http://example.com" 2102 2103 env string // HTTP_PROXY 2104 httpsenv string // HTTPS_PROXY 2105 noenv string // NO_PROXY 2106 reqmeth string // REQUEST_METHOD 2107 2108 want string 2109 wanterr error 2110 } 2111 2112 func (t proxyFromEnvTest) String() string { 2113 var buf bytes.Buffer 2114 space := func() { 2115 if buf.Len() > 0 { 2116 buf.WriteByte(' ') 2117 } 2118 } 2119 if t.env != "" { 2120 fmt.Fprintf(&buf, "http_proxy=%q", t.env) 2121 } 2122 if t.httpsenv != "" { 2123 space() 2124 fmt.Fprintf(&buf, "https_proxy=%q", t.httpsenv) 2125 } 2126 if t.noenv != "" { 2127 space() 2128 fmt.Fprintf(&buf, "no_proxy=%q", t.noenv) 2129 } 2130 if t.reqmeth != "" { 2131 space() 2132 fmt.Fprintf(&buf, "request_method=%q", t.reqmeth) 2133 } 2134 req := "http://example.com" 2135 if t.req != "" { 2136 req = t.req 2137 } 2138 space() 2139 fmt.Fprintf(&buf, "req=%q", req) 2140 return strings.TrimSpace(buf.String()) 2141 } 2142 2143 var proxyFromEnvTests = []proxyFromEnvTest{ 2144 {env: "127.0.0.1:8080", want: "http://127.0.0.1:8080"}, 2145 {env: "cache.corp.example.com:1234", want: "http://cache.corp.example.com:1234"}, 2146 {env: "cache.corp.example.com", want: "http://cache.corp.example.com"}, 2147 {env: "https://cache.corp.example.com", want: "https://cache.corp.example.com"}, 2148 {env: "http://127.0.0.1:8080", want: "http://127.0.0.1:8080"}, 2149 {env: "https://127.0.0.1:8080", want: "https://127.0.0.1:8080"}, 2150 2151 // Don't use secure for http 2152 {req: "http://insecure.tld/", env: "http.proxy.tld", httpsenv: "secure.proxy.tld", want: "http://http.proxy.tld"}, 2153 // Use secure for https. 2154 {req: "https://secure.tld/", env: "http.proxy.tld", httpsenv: "secure.proxy.tld", want: "http://secure.proxy.tld"}, 2155 {req: "https://secure.tld/", env: "http.proxy.tld", httpsenv: "https://secure.proxy.tld", want: "https://secure.proxy.tld"}, 2156 2157 // Issue 16405: don't use HTTP_PROXY in a CGI environment, 2158 // where HTTP_PROXY can be attacker-controlled. 2159 {env: "http://10.1.2.3:8080", reqmeth: "POST", 2160 want: "<nil>", 2161 wanterr: errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")}, 2162 2163 {want: "<nil>"}, 2164 2165 {noenv: "example.com", req: "http://example.com/", env: "proxy", want: "<nil>"}, 2166 {noenv: ".example.com", req: "http://example.com/", env: "proxy", want: "<nil>"}, 2167 {noenv: "ample.com", req: "http://example.com/", env: "proxy", want: "http://proxy"}, 2168 {noenv: "example.com", req: "http://foo.example.com/", env: "proxy", want: "<nil>"}, 2169 {noenv: ".foo.com", req: "http://example.com/", env: "proxy", want: "http://proxy"}, 2170 } 2171 2172 func TestProxyFromEnvironment(t *testing.T) { 2173 ResetProxyEnv() 2174 for _, tt := range proxyFromEnvTests { 2175 os.Setenv("HTTP_PROXY", tt.env) 2176 os.Setenv("HTTPS_PROXY", tt.httpsenv) 2177 os.Setenv("NO_PROXY", tt.noenv) 2178 os.Setenv("REQUEST_METHOD", tt.reqmeth) 2179 ResetCachedEnvironment() 2180 reqURL := tt.req 2181 if reqURL == "" { 2182 reqURL = "http://example.com" 2183 } 2184 req, _ := NewRequest("GET", reqURL, nil) 2185 url, err := ProxyFromEnvironment(req) 2186 if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.wanterr); g != e { 2187 t.Errorf("%v: got error = %q, want %q", tt, g, e) 2188 continue 2189 } 2190 if got := fmt.Sprintf("%s", url); got != tt.want { 2191 t.Errorf("%v: got URL = %q, want %q", tt, url, tt.want) 2192 } 2193 } 2194 } 2195 2196 func TestIdleConnChannelLeak(t *testing.T) { 2197 var mu sync.Mutex 2198 var n int 2199 2200 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2201 mu.Lock() 2202 n++ 2203 mu.Unlock() 2204 })) 2205 defer ts.Close() 2206 2207 const nReqs = 5 2208 didRead := make(chan bool, nReqs) 2209 SetReadLoopBeforeNextReadHook(func() { didRead <- true }) 2210 defer SetReadLoopBeforeNextReadHook(nil) 2211 2212 tr := &Transport{ 2213 Dial: func(netw, addr string) (net.Conn, error) { 2214 return net.Dial(netw, ts.Listener.Addr().String()) 2215 }, 2216 } 2217 defer tr.CloseIdleConnections() 2218 2219 c := &Client{Transport: tr} 2220 2221 // First, without keep-alives. 2222 for _, disableKeep := range []bool{true, false} { 2223 tr.DisableKeepAlives = disableKeep 2224 for i := 0; i < nReqs; i++ { 2225 _, err := c.Get(fmt.Sprintf("http://foo-host-%d.tld/", i)) 2226 if err != nil { 2227 t.Fatal(err) 2228 } 2229 // Note: no res.Body.Close is needed here, since the 2230 // response Content-Length is zero. Perhaps the test 2231 // should be more explicit and use a HEAD, but tests 2232 // elsewhere guarantee that zero byte responses generate 2233 // a "Content-Length: 0" instead of chunking. 2234 } 2235 2236 // At this point, each of the 5 Transport.readLoop goroutines 2237 // are scheduling noting that there are no response bodies (see 2238 // earlier comment), and are then calling putIdleConn, which 2239 // decrements this count. Usually that happens quickly, which is 2240 // why this test has seemed to work for ages. But it's still 2241 // racey: we have wait for them to finish first. See Issue 10427 2242 for i := 0; i < nReqs; i++ { 2243 <-didRead 2244 } 2245 2246 if got := tr.IdleConnChMapSizeForTesting(); got != 0 { 2247 t.Fatalf("ForDisableKeepAlives = %v, map size = %d; want 0", disableKeep, got) 2248 } 2249 } 2250 } 2251 2252 // Verify the status quo: that the Client.Post function coerces its 2253 // body into a ReadCloser if it's a Closer, and that the Transport 2254 // then closes it. 2255 func TestTransportClosesRequestBody(t *testing.T) { 2256 defer afterTest(t) 2257 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2258 io.Copy(ioutil.Discard, r.Body) 2259 })) 2260 defer ts.Close() 2261 2262 tr := &Transport{} 2263 defer tr.CloseIdleConnections() 2264 cl := &Client{Transport: tr} 2265 2266 closes := 0 2267 2268 res, err := cl.Post(ts.URL, "text/plain", countCloseReader{&closes, strings.NewReader("hello")}) 2269 if err != nil { 2270 t.Fatal(err) 2271 } 2272 res.Body.Close() 2273 if closes != 1 { 2274 t.Errorf("closes = %d; want 1", closes) 2275 } 2276 } 2277 2278 func TestTransportTLSHandshakeTimeout(t *testing.T) { 2279 defer afterTest(t) 2280 if testing.Short() { 2281 t.Skip("skipping in short mode") 2282 } 2283 ln := newLocalListener(t) 2284 defer ln.Close() 2285 testdonec := make(chan struct{}) 2286 defer close(testdonec) 2287 2288 go func() { 2289 c, err := ln.Accept() 2290 if err != nil { 2291 t.Error(err) 2292 return 2293 } 2294 <-testdonec 2295 c.Close() 2296 }() 2297 2298 getdonec := make(chan struct{}) 2299 go func() { 2300 defer close(getdonec) 2301 tr := &Transport{ 2302 Dial: func(_, _ string) (net.Conn, error) { 2303 return net.Dial("tcp", ln.Addr().String()) 2304 }, 2305 TLSHandshakeTimeout: 250 * time.Millisecond, 2306 } 2307 cl := &Client{Transport: tr} 2308 _, err := cl.Get("https://dummy.tld/") 2309 if err == nil { 2310 t.Error("expected error") 2311 return 2312 } 2313 ue, ok := err.(*url.Error) 2314 if !ok { 2315 t.Errorf("expected url.Error; got %#v", err) 2316 return 2317 } 2318 ne, ok := ue.Err.(net.Error) 2319 if !ok { 2320 t.Errorf("expected net.Error; got %#v", err) 2321 return 2322 } 2323 if !ne.Timeout() { 2324 t.Errorf("expected timeout error; got %v", err) 2325 } 2326 if !strings.Contains(err.Error(), "handshake timeout") { 2327 t.Errorf("expected 'handshake timeout' in error; got %v", err) 2328 } 2329 }() 2330 select { 2331 case <-getdonec: 2332 case <-time.After(5 * time.Second): 2333 t.Error("test timeout; TLS handshake hung?") 2334 } 2335 } 2336 2337 // Trying to repro golang.org/issue/3514 2338 func TestTLSServerClosesConnection(t *testing.T) { 2339 defer afterTest(t) 2340 testenv.SkipFlaky(t, 7634) 2341 2342 closedc := make(chan bool, 1) 2343 ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2344 if strings.Contains(r.URL.Path, "/keep-alive-then-die") { 2345 conn, _, _ := w.(Hijacker).Hijack() 2346 conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nfoo")) 2347 conn.Close() 2348 closedc <- true 2349 return 2350 } 2351 fmt.Fprintf(w, "hello") 2352 })) 2353 defer ts.Close() 2354 tr := &Transport{ 2355 TLSClientConfig: &tls.Config{ 2356 InsecureSkipVerify: true, 2357 }, 2358 } 2359 defer tr.CloseIdleConnections() 2360 client := &Client{Transport: tr} 2361 2362 var nSuccess = 0 2363 var errs []error 2364 const trials = 20 2365 for i := 0; i < trials; i++ { 2366 tr.CloseIdleConnections() 2367 res, err := client.Get(ts.URL + "/keep-alive-then-die") 2368 if err != nil { 2369 t.Fatal(err) 2370 } 2371 <-closedc 2372 slurp, err := ioutil.ReadAll(res.Body) 2373 if err != nil { 2374 t.Fatal(err) 2375 } 2376 if string(slurp) != "foo" { 2377 t.Errorf("Got %q, want foo", slurp) 2378 } 2379 2380 // Now try again and see if we successfully 2381 // pick a new connection. 2382 res, err = client.Get(ts.URL + "/") 2383 if err != nil { 2384 errs = append(errs, err) 2385 continue 2386 } 2387 slurp, err = ioutil.ReadAll(res.Body) 2388 if err != nil { 2389 errs = append(errs, err) 2390 continue 2391 } 2392 nSuccess++ 2393 } 2394 if nSuccess > 0 { 2395 t.Logf("successes = %d of %d", nSuccess, trials) 2396 } else { 2397 t.Errorf("All runs failed:") 2398 } 2399 for _, err := range errs { 2400 t.Logf(" err: %v", err) 2401 } 2402 } 2403 2404 // byteFromChanReader is an io.Reader that reads a single byte at a 2405 // time from the channel. When the channel is closed, the reader 2406 // returns io.EOF. 2407 type byteFromChanReader chan byte 2408 2409 func (c byteFromChanReader) Read(p []byte) (n int, err error) { 2410 if len(p) == 0 { 2411 return 2412 } 2413 b, ok := <-c 2414 if !ok { 2415 return 0, io.EOF 2416 } 2417 p[0] = b 2418 return 1, nil 2419 } 2420 2421 // Verifies that the Transport doesn't reuse a connection in the case 2422 // where the server replies before the request has been fully 2423 // written. We still honor that reply (see TestIssue3595), but don't 2424 // send future requests on the connection because it's then in a 2425 // questionable state. 2426 // golang.org/issue/7569 2427 func TestTransportNoReuseAfterEarlyResponse(t *testing.T) { 2428 defer afterTest(t) 2429 var sconn struct { 2430 sync.Mutex 2431 c net.Conn 2432 } 2433 var getOkay bool 2434 closeConn := func() { 2435 sconn.Lock() 2436 defer sconn.Unlock() 2437 if sconn.c != nil { 2438 sconn.c.Close() 2439 sconn.c = nil 2440 if !getOkay { 2441 t.Logf("Closed server connection") 2442 } 2443 } 2444 } 2445 defer closeConn() 2446 2447 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2448 if r.Method == "GET" { 2449 io.WriteString(w, "bar") 2450 return 2451 } 2452 conn, _, _ := w.(Hijacker).Hijack() 2453 sconn.Lock() 2454 sconn.c = conn 2455 sconn.Unlock() 2456 conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nfoo")) // keep-alive 2457 go io.Copy(ioutil.Discard, conn) 2458 })) 2459 defer ts.Close() 2460 tr := &Transport{} 2461 defer tr.CloseIdleConnections() 2462 client := &Client{Transport: tr} 2463 2464 const bodySize = 256 << 10 2465 finalBit := make(byteFromChanReader, 1) 2466 req, _ := NewRequest("POST", ts.URL, io.MultiReader(io.LimitReader(neverEnding('x'), bodySize-1), finalBit)) 2467 req.ContentLength = bodySize 2468 res, err := client.Do(req) 2469 if err := wantBody(res, err, "foo"); err != nil { 2470 t.Errorf("POST response: %v", err) 2471 } 2472 donec := make(chan bool) 2473 go func() { 2474 defer close(donec) 2475 res, err = client.Get(ts.URL) 2476 if err := wantBody(res, err, "bar"); err != nil { 2477 t.Errorf("GET response: %v", err) 2478 return 2479 } 2480 getOkay = true // suppress test noise 2481 }() 2482 time.AfterFunc(5*time.Second, closeConn) 2483 select { 2484 case <-donec: 2485 finalBit <- 'x' // unblock the writeloop of the first Post 2486 close(finalBit) 2487 case <-time.After(7 * time.Second): 2488 t.Fatal("timeout waiting for GET request to finish") 2489 } 2490 } 2491 2492 // Tests that we don't leak Transport persistConn.readLoop goroutines 2493 // when a server hangs up immediately after saying it would keep-alive. 2494 func TestTransportIssue10457(t *testing.T) { 2495 defer afterTest(t) // used to fail in goroutine leak check 2496 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2497 // Send a response with no body, keep-alive 2498 // (implicit), and then lie and immediately close the 2499 // connection. This forces the Transport's readLoop to 2500 // immediately Peek an io.EOF and get to the point 2501 // that used to hang. 2502 conn, _, _ := w.(Hijacker).Hijack() 2503 conn.Write([]byte("HTTP/1.1 200 OK\r\nFoo: Bar\r\nContent-Length: 0\r\n\r\n")) // keep-alive 2504 conn.Close() 2505 })) 2506 defer ts.Close() 2507 tr := &Transport{} 2508 defer tr.CloseIdleConnections() 2509 cl := &Client{Transport: tr} 2510 res, err := cl.Get(ts.URL) 2511 if err != nil { 2512 t.Fatalf("Get: %v", err) 2513 } 2514 defer res.Body.Close() 2515 2516 // Just a sanity check that we at least get the response. The real 2517 // test here is that the "defer afterTest" above doesn't find any 2518 // leaked goroutines. 2519 if got, want := res.Header.Get("Foo"), "Bar"; got != want { 2520 t.Errorf("Foo header = %q; want %q", got, want) 2521 } 2522 } 2523 2524 type errorReader struct { 2525 err error 2526 } 2527 2528 func (e errorReader) Read(p []byte) (int, error) { return 0, e.err } 2529 2530 type closerFunc func() error 2531 2532 func (f closerFunc) Close() error { return f() } 2533 2534 // Issue 4677. If we try to reuse a connection that the server is in the 2535 // process of closing, we may end up successfully writing out our request (or a 2536 // portion of our request) only to find a connection error when we try to read 2537 // from (or finish writing to) the socket. 2538 // 2539 // NOTE: we resend a request only if the request is idempotent, we reused a 2540 // keep-alive connection, and we haven't yet received any header data. This 2541 // automatically prevents an infinite resend loop because we'll run out of the 2542 // cached keep-alive connections eventually. 2543 func TestRetryIdempotentRequestsOnError(t *testing.T) { 2544 defer afterTest(t) 2545 2546 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2547 })) 2548 defer ts.Close() 2549 2550 tr := &Transport{} 2551 c := &Client{Transport: tr} 2552 2553 const N = 2 2554 retryc := make(chan struct{}, N) 2555 SetRoundTripRetried(func() { 2556 retryc <- struct{}{} 2557 }) 2558 defer SetRoundTripRetried(nil) 2559 2560 for n := 0; n < 100; n++ { 2561 // open 2 conns 2562 errc := make(chan error, N) 2563 for i := 0; i < N; i++ { 2564 // start goroutines, send on errc 2565 go func() { 2566 res, err := c.Get(ts.URL) 2567 if err == nil { 2568 res.Body.Close() 2569 } 2570 errc <- err 2571 }() 2572 } 2573 for i := 0; i < N; i++ { 2574 if err := <-errc; err != nil { 2575 t.Fatal(err) 2576 } 2577 } 2578 2579 ts.CloseClientConnections() 2580 for i := 0; i < N; i++ { 2581 go func() { 2582 res, err := c.Get(ts.URL) 2583 if err == nil { 2584 res.Body.Close() 2585 } 2586 errc <- err 2587 }() 2588 } 2589 2590 for i := 0; i < N; i++ { 2591 if err := <-errc; err != nil { 2592 t.Fatal(err) 2593 } 2594 } 2595 for i := 0; i < N; i++ { 2596 select { 2597 case <-retryc: 2598 // we triggered a retry, test was successful 2599 t.Logf("finished after %d runs\n", n) 2600 return 2601 default: 2602 } 2603 } 2604 } 2605 t.Fatal("did not trigger any retries") 2606 } 2607 2608 // Issue 6981 2609 func TestTransportClosesBodyOnError(t *testing.T) { 2610 setParallel(t) 2611 defer afterTest(t) 2612 readBody := make(chan error, 1) 2613 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2614 _, err := ioutil.ReadAll(r.Body) 2615 readBody <- err 2616 })) 2617 defer ts.Close() 2618 fakeErr := errors.New("fake error") 2619 didClose := make(chan bool, 1) 2620 req, _ := NewRequest("POST", ts.URL, struct { 2621 io.Reader 2622 io.Closer 2623 }{ 2624 io.MultiReader(io.LimitReader(neverEnding('x'), 1<<20), errorReader{fakeErr}), 2625 closerFunc(func() error { 2626 select { 2627 case didClose <- true: 2628 default: 2629 } 2630 return nil 2631 }), 2632 }) 2633 res, err := DefaultClient.Do(req) 2634 if res != nil { 2635 defer res.Body.Close() 2636 } 2637 if err == nil || !strings.Contains(err.Error(), fakeErr.Error()) { 2638 t.Fatalf("Do error = %v; want something containing %q", err, fakeErr.Error()) 2639 } 2640 select { 2641 case err := <-readBody: 2642 if err == nil { 2643 t.Errorf("Unexpected success reading request body from handler; want 'unexpected EOF reading trailer'") 2644 } 2645 case <-time.After(5 * time.Second): 2646 t.Error("timeout waiting for server handler to complete") 2647 } 2648 select { 2649 case <-didClose: 2650 default: 2651 t.Errorf("didn't see Body.Close") 2652 } 2653 } 2654 2655 func TestTransportDialTLS(t *testing.T) { 2656 var mu sync.Mutex // guards following 2657 var gotReq, didDial bool 2658 2659 ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2660 mu.Lock() 2661 gotReq = true 2662 mu.Unlock() 2663 })) 2664 defer ts.Close() 2665 tr := &Transport{ 2666 DialTLS: func(netw, addr string) (net.Conn, error) { 2667 mu.Lock() 2668 didDial = true 2669 mu.Unlock() 2670 c, err := tls.Dial(netw, addr, &tls.Config{ 2671 InsecureSkipVerify: true, 2672 }) 2673 if err != nil { 2674 return nil, err 2675 } 2676 return c, c.Handshake() 2677 }, 2678 } 2679 defer tr.CloseIdleConnections() 2680 client := &Client{Transport: tr} 2681 res, err := client.Get(ts.URL) 2682 if err != nil { 2683 t.Fatal(err) 2684 } 2685 res.Body.Close() 2686 mu.Lock() 2687 if !gotReq { 2688 t.Error("didn't get request") 2689 } 2690 if !didDial { 2691 t.Error("didn't use dial hook") 2692 } 2693 } 2694 2695 // Test for issue 8755 2696 // Ensure that if a proxy returns an error, it is exposed by RoundTrip 2697 func TestRoundTripReturnsProxyError(t *testing.T) { 2698 badProxy := func(*Request) (*url.URL, error) { 2699 return nil, errors.New("errorMessage") 2700 } 2701 2702 tr := &Transport{Proxy: badProxy} 2703 2704 req, _ := NewRequest("GET", "http://example.com", nil) 2705 2706 _, err := tr.RoundTrip(req) 2707 2708 if err == nil { 2709 t.Error("Expected proxy error to be returned by RoundTrip") 2710 } 2711 } 2712 2713 // tests that putting an idle conn after a call to CloseIdleConns does return it 2714 func TestTransportCloseIdleConnsThenReturn(t *testing.T) { 2715 tr := &Transport{} 2716 wantIdle := func(when string, n int) bool { 2717 got := tr.IdleConnCountForTesting("|http|example.com") // key used by PutIdleTestConn 2718 if got == n { 2719 return true 2720 } 2721 t.Errorf("%s: idle conns = %d; want %d", when, got, n) 2722 return false 2723 } 2724 wantIdle("start", 0) 2725 if !tr.PutIdleTestConn() { 2726 t.Fatal("put failed") 2727 } 2728 if !tr.PutIdleTestConn() { 2729 t.Fatal("second put failed") 2730 } 2731 wantIdle("after put", 2) 2732 tr.CloseIdleConnections() 2733 if !tr.IsIdleForTesting() { 2734 t.Error("should be idle after CloseIdleConnections") 2735 } 2736 wantIdle("after close idle", 0) 2737 if tr.PutIdleTestConn() { 2738 t.Fatal("put didn't fail") 2739 } 2740 wantIdle("after second put", 0) 2741 2742 tr.RequestIdleConnChForTesting() // should toggle the transport out of idle mode 2743 if tr.IsIdleForTesting() { 2744 t.Error("shouldn't be idle after RequestIdleConnChForTesting") 2745 } 2746 if !tr.PutIdleTestConn() { 2747 t.Fatal("after re-activation") 2748 } 2749 wantIdle("after final put", 1) 2750 } 2751 2752 // This tests that an client requesting a content range won't also 2753 // implicitly ask for gzip support. If they want that, they need to do it 2754 // on their own. 2755 // golang.org/issue/8923 2756 func TestTransportRangeAndGzip(t *testing.T) { 2757 defer afterTest(t) 2758 reqc := make(chan *Request, 1) 2759 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2760 reqc <- r 2761 })) 2762 defer ts.Close() 2763 2764 req, _ := NewRequest("GET", ts.URL, nil) 2765 req.Header.Set("Range", "bytes=7-11") 2766 res, err := DefaultClient.Do(req) 2767 if err != nil { 2768 t.Fatal(err) 2769 } 2770 2771 select { 2772 case r := <-reqc: 2773 if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { 2774 t.Error("Transport advertised gzip support in the Accept header") 2775 } 2776 if r.Header.Get("Range") == "" { 2777 t.Error("no Range in request") 2778 } 2779 case <-time.After(10 * time.Second): 2780 t.Fatal("timeout") 2781 } 2782 res.Body.Close() 2783 } 2784 2785 // Test for issue 10474 2786 func TestTransportResponseCancelRace(t *testing.T) { 2787 defer afterTest(t) 2788 2789 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2790 // important that this response has a body. 2791 var b [1024]byte 2792 w.Write(b[:]) 2793 })) 2794 defer ts.Close() 2795 2796 tr := &Transport{} 2797 defer tr.CloseIdleConnections() 2798 2799 req, err := NewRequest("GET", ts.URL, nil) 2800 if err != nil { 2801 t.Fatal(err) 2802 } 2803 res, err := tr.RoundTrip(req) 2804 if err != nil { 2805 t.Fatal(err) 2806 } 2807 // If we do an early close, Transport just throws the connection away and 2808 // doesn't reuse it. In order to trigger the bug, it has to reuse the connection 2809 // so read the body 2810 if _, err := io.Copy(ioutil.Discard, res.Body); err != nil { 2811 t.Fatal(err) 2812 } 2813 2814 req2, err := NewRequest("GET", ts.URL, nil) 2815 if err != nil { 2816 t.Fatal(err) 2817 } 2818 tr.CancelRequest(req) 2819 res, err = tr.RoundTrip(req2) 2820 if err != nil { 2821 t.Fatal(err) 2822 } 2823 res.Body.Close() 2824 } 2825 2826 func TestTransportDialCancelRace(t *testing.T) { 2827 defer afterTest(t) 2828 2829 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) 2830 defer ts.Close() 2831 2832 tr := &Transport{} 2833 defer tr.CloseIdleConnections() 2834 2835 req, err := NewRequest("GET", ts.URL, nil) 2836 if err != nil { 2837 t.Fatal(err) 2838 } 2839 SetEnterRoundTripHook(func() { 2840 tr.CancelRequest(req) 2841 }) 2842 defer SetEnterRoundTripHook(nil) 2843 res, err := tr.RoundTrip(req) 2844 if err != ExportErrRequestCanceled { 2845 t.Errorf("expected canceled request error; got %v", err) 2846 if err == nil { 2847 res.Body.Close() 2848 } 2849 } 2850 } 2851 2852 // logWritesConn is a net.Conn that logs each Write call to writes 2853 // and then proxies to w. 2854 // It proxies Read calls to a reader it receives from rch. 2855 type logWritesConn struct { 2856 net.Conn // nil. crash on use. 2857 2858 w io.Writer 2859 2860 rch <-chan io.Reader 2861 r io.Reader // nil until received by rch 2862 2863 mu sync.Mutex 2864 writes []string 2865 } 2866 2867 func (c *logWritesConn) Write(p []byte) (n int, err error) { 2868 c.mu.Lock() 2869 defer c.mu.Unlock() 2870 c.writes = append(c.writes, string(p)) 2871 return c.w.Write(p) 2872 } 2873 2874 func (c *logWritesConn) Read(p []byte) (n int, err error) { 2875 if c.r == nil { 2876 c.r = <-c.rch 2877 } 2878 return c.r.Read(p) 2879 } 2880 2881 func (c *logWritesConn) Close() error { return nil } 2882 2883 // Issue 6574 2884 func TestTransportFlushesBodyChunks(t *testing.T) { 2885 defer afterTest(t) 2886 resBody := make(chan io.Reader, 1) 2887 connr, connw := io.Pipe() // connection pipe pair 2888 lw := &logWritesConn{ 2889 rch: resBody, 2890 w: connw, 2891 } 2892 tr := &Transport{ 2893 Dial: func(network, addr string) (net.Conn, error) { 2894 return lw, nil 2895 }, 2896 } 2897 bodyr, bodyw := io.Pipe() // body pipe pair 2898 go func() { 2899 defer bodyw.Close() 2900 for i := 0; i < 3; i++ { 2901 fmt.Fprintf(bodyw, "num%d\n", i) 2902 } 2903 }() 2904 resc := make(chan *Response) 2905 go func() { 2906 req, _ := NewRequest("POST", "http://localhost:8080", bodyr) 2907 req.Header.Set("User-Agent", "x") // known value for test 2908 res, err := tr.RoundTrip(req) 2909 if err != nil { 2910 t.Errorf("RoundTrip: %v", err) 2911 close(resc) 2912 return 2913 } 2914 resc <- res 2915 2916 }() 2917 // Fully consume the request before checking the Write log vs. want. 2918 req, err := ReadRequest(bufio.NewReader(connr)) 2919 if err != nil { 2920 t.Fatal(err) 2921 } 2922 io.Copy(ioutil.Discard, req.Body) 2923 2924 // Unblock the transport's roundTrip goroutine. 2925 resBody <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") 2926 res, ok := <-resc 2927 if !ok { 2928 return 2929 } 2930 defer res.Body.Close() 2931 2932 want := []string{ 2933 "POST / HTTP/1.1\r\nHost: localhost:8080\r\nUser-Agent: x\r\nTransfer-Encoding: chunked\r\nAccept-Encoding: gzip\r\n\r\n" + 2934 "5\r\nnum0\n\r\n", 2935 "5\r\nnum1\n\r\n", 2936 "5\r\nnum2\n\r\n", 2937 "0\r\n\r\n", 2938 } 2939 if !reflect.DeepEqual(lw.writes, want) { 2940 t.Errorf("Writes differed.\n Got: %q\nWant: %q\n", lw.writes, want) 2941 } 2942 } 2943 2944 // Issue 11745. 2945 func TestTransportPrefersResponseOverWriteError(t *testing.T) { 2946 if testing.Short() { 2947 t.Skip("skipping in short mode") 2948 } 2949 defer afterTest(t) 2950 const contentLengthLimit = 1024 * 1024 // 1MB 2951 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 2952 if r.ContentLength >= contentLengthLimit { 2953 w.WriteHeader(StatusBadRequest) 2954 r.Body.Close() 2955 return 2956 } 2957 w.WriteHeader(StatusOK) 2958 })) 2959 defer ts.Close() 2960 2961 fail := 0 2962 count := 100 2963 bigBody := strings.Repeat("a", contentLengthLimit*2) 2964 for i := 0; i < count; i++ { 2965 req, err := NewRequest("PUT", ts.URL, strings.NewReader(bigBody)) 2966 if err != nil { 2967 t.Fatal(err) 2968 } 2969 tr := new(Transport) 2970 defer tr.CloseIdleConnections() 2971 client := &Client{Transport: tr} 2972 resp, err := client.Do(req) 2973 if err != nil { 2974 fail++ 2975 t.Logf("%d = %#v", i, err) 2976 if ue, ok := err.(*url.Error); ok { 2977 t.Logf("urlErr = %#v", ue.Err) 2978 if ne, ok := ue.Err.(*net.OpError); ok { 2979 t.Logf("netOpError = %#v", ne.Err) 2980 } 2981 } 2982 } else { 2983 resp.Body.Close() 2984 if resp.StatusCode != 400 { 2985 t.Errorf("Expected status code 400, got %v", resp.Status) 2986 } 2987 } 2988 } 2989 if fail > 0 { 2990 t.Errorf("Failed %v out of %v\n", fail, count) 2991 } 2992 } 2993 2994 func TestTransportAutomaticHTTP2(t *testing.T) { 2995 testTransportAutoHTTP(t, &Transport{}, true) 2996 } 2997 2998 // golang.org/issue/14391: also check DefaultTransport 2999 func TestTransportAutomaticHTTP2_DefaultTransport(t *testing.T) { 3000 testTransportAutoHTTP(t, DefaultTransport.(*Transport), true) 3001 } 3002 3003 func TestTransportAutomaticHTTP2_TLSNextProto(t *testing.T) { 3004 testTransportAutoHTTP(t, &Transport{ 3005 TLSNextProto: make(map[string]func(string, *tls.Conn) RoundTripper), 3006 }, false) 3007 } 3008 3009 func TestTransportAutomaticHTTP2_TLSConfig(t *testing.T) { 3010 testTransportAutoHTTP(t, &Transport{ 3011 TLSClientConfig: new(tls.Config), 3012 }, false) 3013 } 3014 3015 func TestTransportAutomaticHTTP2_ExpectContinueTimeout(t *testing.T) { 3016 testTransportAutoHTTP(t, &Transport{ 3017 ExpectContinueTimeout: 1 * time.Second, 3018 }, true) 3019 } 3020 3021 func TestTransportAutomaticHTTP2_Dial(t *testing.T) { 3022 var d net.Dialer 3023 testTransportAutoHTTP(t, &Transport{ 3024 Dial: d.Dial, 3025 }, false) 3026 } 3027 3028 func TestTransportAutomaticHTTP2_DialTLS(t *testing.T) { 3029 testTransportAutoHTTP(t, &Transport{ 3030 DialTLS: func(network, addr string) (net.Conn, error) { 3031 panic("unused") 3032 }, 3033 }, false) 3034 } 3035 3036 func testTransportAutoHTTP(t *testing.T, tr *Transport, wantH2 bool) { 3037 _, err := tr.RoundTrip(new(Request)) 3038 if err == nil { 3039 t.Error("expected error from RoundTrip") 3040 } 3041 if reg := tr.TLSNextProto["h2"] != nil; reg != wantH2 { 3042 t.Errorf("HTTP/2 registered = %v; want %v", reg, wantH2) 3043 } 3044 } 3045 3046 // Issue 13633: there was a race where we returned bodyless responses 3047 // to callers before recycling the persistent connection, which meant 3048 // a client doing two subsequent requests could end up on different 3049 // connections. It's somewhat harmless but enough tests assume it's 3050 // not true in order to test other things that it's worth fixing. 3051 // Plus it's nice to be consistent and not have timing-dependent 3052 // behavior. 3053 func TestTransportReuseConnEmptyResponseBody(t *testing.T) { 3054 defer afterTest(t) 3055 cst := newClientServerTest(t, h1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { 3056 w.Header().Set("X-Addr", r.RemoteAddr) 3057 // Empty response body. 3058 })) 3059 defer cst.close() 3060 n := 100 3061 if testing.Short() { 3062 n = 10 3063 } 3064 var firstAddr string 3065 for i := 0; i < n; i++ { 3066 res, err := cst.c.Get(cst.ts.URL) 3067 if err != nil { 3068 log.Fatal(err) 3069 } 3070 addr := res.Header.Get("X-Addr") 3071 if i == 0 { 3072 firstAddr = addr 3073 } else if addr != firstAddr { 3074 t.Fatalf("On request %d, addr %q != original addr %q", i+1, addr, firstAddr) 3075 } 3076 res.Body.Close() 3077 } 3078 } 3079 3080 // Issue 13839 3081 func TestNoCrashReturningTransportAltConn(t *testing.T) { 3082 cert, err := tls.X509KeyPair(internal.LocalhostCert, internal.LocalhostKey) 3083 if err != nil { 3084 t.Fatal(err) 3085 } 3086 ln := newLocalListener(t) 3087 defer ln.Close() 3088 3089 handledPendingDial := make(chan bool, 1) 3090 SetPendingDialHooks(nil, func() { handledPendingDial <- true }) 3091 defer SetPendingDialHooks(nil, nil) 3092 3093 testDone := make(chan struct{}) 3094 defer close(testDone) 3095 go func() { 3096 tln := tls.NewListener(ln, &tls.Config{ 3097 NextProtos: []string{"foo"}, 3098 Certificates: []tls.Certificate{cert}, 3099 }) 3100 sc, err := tln.Accept() 3101 if err != nil { 3102 t.Error(err) 3103 return 3104 } 3105 if err := sc.(*tls.Conn).Handshake(); err != nil { 3106 t.Error(err) 3107 return 3108 } 3109 <-testDone 3110 sc.Close() 3111 }() 3112 3113 addr := ln.Addr().String() 3114 3115 req, _ := NewRequest("GET", "https://fake.tld/", nil) 3116 cancel := make(chan struct{}) 3117 req.Cancel = cancel 3118 3119 doReturned := make(chan bool, 1) 3120 madeRoundTripper := make(chan bool, 1) 3121 3122 tr := &Transport{ 3123 DisableKeepAlives: true, 3124 TLSNextProto: map[string]func(string, *tls.Conn) RoundTripper{ 3125 "foo": func(authority string, c *tls.Conn) RoundTripper { 3126 madeRoundTripper <- true 3127 return funcRoundTripper(func() { 3128 t.Error("foo RoundTripper should not be called") 3129 }) 3130 }, 3131 }, 3132 Dial: func(_, _ string) (net.Conn, error) { 3133 panic("shouldn't be called") 3134 }, 3135 DialTLS: func(_, _ string) (net.Conn, error) { 3136 tc, err := tls.Dial("tcp", addr, &tls.Config{ 3137 InsecureSkipVerify: true, 3138 NextProtos: []string{"foo"}, 3139 }) 3140 if err != nil { 3141 return nil, err 3142 } 3143 if err := tc.Handshake(); err != nil { 3144 return nil, err 3145 } 3146 close(cancel) 3147 <-doReturned 3148 return tc, nil 3149 }, 3150 } 3151 c := &Client{Transport: tr} 3152 3153 _, err = c.Do(req) 3154 if ue, ok := err.(*url.Error); !ok || ue.Err != ExportErrRequestCanceledConn { 3155 t.Fatalf("Do error = %v; want url.Error with errRequestCanceledConn", err) 3156 } 3157 3158 doReturned <- true 3159 <-madeRoundTripper 3160 <-handledPendingDial 3161 } 3162 3163 func TestTransportReuseConnection_Gzip_Chunked(t *testing.T) { 3164 testTransportReuseConnection_Gzip(t, true) 3165 } 3166 3167 func TestTransportReuseConnection_Gzip_ContentLength(t *testing.T) { 3168 testTransportReuseConnection_Gzip(t, false) 3169 } 3170 3171 // Make sure we re-use underlying TCP connection for gzipped responses too. 3172 func testTransportReuseConnection_Gzip(t *testing.T, chunked bool) { 3173 defer afterTest(t) 3174 addr := make(chan string, 2) 3175 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 3176 addr <- r.RemoteAddr 3177 w.Header().Set("Content-Encoding", "gzip") 3178 if chunked { 3179 w.(Flusher).Flush() 3180 } 3181 w.Write(rgz) // arbitrary gzip response 3182 })) 3183 defer ts.Close() 3184 3185 tr := &Transport{} 3186 defer tr.CloseIdleConnections() 3187 c := &Client{Transport: tr} 3188 for i := 0; i < 2; i++ { 3189 res, err := c.Get(ts.URL) 3190 if err != nil { 3191 t.Fatal(err) 3192 } 3193 buf := make([]byte, len(rgz)) 3194 if n, err := io.ReadFull(res.Body, buf); err != nil { 3195 t.Errorf("%d. ReadFull = %v, %v", i, n, err) 3196 } 3197 // Note: no res.Body.Close call. It should work without it, 3198 // since the flate.Reader's internal buffering will hit EOF 3199 // and that should be sufficient. 3200 } 3201 a1, a2 := <-addr, <-addr 3202 if a1 != a2 { 3203 t.Fatalf("didn't reuse connection") 3204 } 3205 } 3206 3207 func TestTransportResponseHeaderLength(t *testing.T) { 3208 defer afterTest(t) 3209 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 3210 if r.URL.Path == "/long" { 3211 w.Header().Set("Long", strings.Repeat("a", 1<<20)) 3212 } 3213 })) 3214 defer ts.Close() 3215 3216 tr := &Transport{ 3217 MaxResponseHeaderBytes: 512 << 10, 3218 } 3219 defer tr.CloseIdleConnections() 3220 c := &Client{Transport: tr} 3221 if res, err := c.Get(ts.URL); err != nil { 3222 t.Fatal(err) 3223 } else { 3224 res.Body.Close() 3225 } 3226 3227 res, err := c.Get(ts.URL + "/long") 3228 if err == nil { 3229 defer res.Body.Close() 3230 var n int64 3231 for k, vv := range res.Header { 3232 for _, v := range vv { 3233 n += int64(len(k)) + int64(len(v)) 3234 } 3235 } 3236 t.Fatalf("Unexpected success. Got %v and %d bytes of response headers", res.Status, n) 3237 } 3238 if want := "server response headers exceeded 524288 bytes"; !strings.Contains(err.Error(), want) { 3239 t.Errorf("got error: %v; want %q", err, want) 3240 } 3241 } 3242 3243 func TestTransportEventTrace(t *testing.T) { testTransportEventTrace(t, h1Mode, false) } 3244 func TestTransportEventTrace_h2(t *testing.T) { testTransportEventTrace(t, h2Mode, false) } 3245 3246 // test a non-nil httptrace.ClientTrace but with all hooks set to zero. 3247 func TestTransportEventTrace_NoHooks(t *testing.T) { testTransportEventTrace(t, h1Mode, true) } 3248 func TestTransportEventTrace_NoHooks_h2(t *testing.T) { testTransportEventTrace(t, h2Mode, true) } 3249 3250 func testTransportEventTrace(t *testing.T, h2 bool, noHooks bool) { 3251 defer afterTest(t) 3252 const resBody = "some body" 3253 gotWroteReqEvent := make(chan struct{}) 3254 cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { 3255 if _, err := ioutil.ReadAll(r.Body); err != nil { 3256 t.Error(err) 3257 } 3258 if !noHooks { 3259 select { 3260 case <-gotWroteReqEvent: 3261 case <-time.After(5 * time.Second): 3262 t.Error("timeout waiting for WroteRequest event") 3263 } 3264 } 3265 io.WriteString(w, resBody) 3266 })) 3267 defer cst.close() 3268 3269 cst.tr.ExpectContinueTimeout = 1 * time.Second 3270 3271 var mu sync.Mutex // guards buf 3272 var buf bytes.Buffer 3273 logf := func(format string, args ...interface{}) { 3274 mu.Lock() 3275 defer mu.Unlock() 3276 fmt.Fprintf(&buf, format, args...) 3277 buf.WriteByte('\n') 3278 } 3279 3280 addrStr := cst.ts.Listener.Addr().String() 3281 ip, port, err := net.SplitHostPort(addrStr) 3282 if err != nil { 3283 t.Fatal(err) 3284 } 3285 3286 // Install a fake DNS server. 3287 ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { 3288 if host != "dns-is-faked.golang" { 3289 t.Errorf("unexpected DNS host lookup for %q", host) 3290 return nil, nil 3291 } 3292 return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil 3293 }) 3294 3295 req, _ := NewRequest("POST", cst.scheme()+"://dns-is-faked.golang:"+port, strings.NewReader("some body")) 3296 trace := &httptrace.ClientTrace{ 3297 GetConn: func(hostPort string) { logf("Getting conn for %v ...", hostPort) }, 3298 GotConn: func(ci httptrace.GotConnInfo) { logf("got conn: %+v", ci) }, 3299 GotFirstResponseByte: func() { logf("first response byte") }, 3300 PutIdleConn: func(err error) { logf("PutIdleConn = %v", err) }, 3301 DNSStart: func(e httptrace.DNSStartInfo) { logf("DNS start: %+v", e) }, 3302 DNSDone: func(e httptrace.DNSDoneInfo) { logf("DNS done: %+v", e) }, 3303 ConnectStart: func(network, addr string) { logf("ConnectStart: Connecting to %s %s ...", network, addr) }, 3304 ConnectDone: func(network, addr string, err error) { 3305 if err != nil { 3306 t.Errorf("ConnectDone: %v", err) 3307 } 3308 logf("ConnectDone: connected to %s %s = %v", network, addr, err) 3309 }, 3310 Wait100Continue: func() { logf("Wait100Continue") }, 3311 Got100Continue: func() { logf("Got100Continue") }, 3312 WroteRequest: func(e httptrace.WroteRequestInfo) { 3313 logf("WroteRequest: %+v", e) 3314 close(gotWroteReqEvent) 3315 }, 3316 } 3317 if h2 { 3318 trace.TLSHandshakeStart = func() { logf("tls handshake start") } 3319 trace.TLSHandshakeDone = func(s tls.ConnectionState, err error) { 3320 logf("tls handshake done. ConnectionState = %v \n err = %v", s, err) 3321 } 3322 } 3323 if noHooks { 3324 // zero out all func pointers, trying to get some path to crash 3325 *trace = httptrace.ClientTrace{} 3326 } 3327 req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) 3328 3329 req.Header.Set("Expect", "100-continue") 3330 res, err := cst.c.Do(req) 3331 if err != nil { 3332 t.Fatal(err) 3333 } 3334 logf("got roundtrip.response") 3335 slurp, err := ioutil.ReadAll(res.Body) 3336 if err != nil { 3337 t.Fatal(err) 3338 } 3339 logf("consumed body") 3340 if string(slurp) != resBody || res.StatusCode != 200 { 3341 t.Fatalf("Got %q, %v; want %q, 200 OK", slurp, res.Status, resBody) 3342 } 3343 res.Body.Close() 3344 3345 if noHooks { 3346 // Done at this point. Just testing a full HTTP 3347 // requests can happen with a trace pointing to a zero 3348 // ClientTrace, full of nil func pointers. 3349 return 3350 } 3351 3352 mu.Lock() 3353 got := buf.String() 3354 mu.Unlock() 3355 3356 wantOnce := func(sub string) { 3357 if strings.Count(got, sub) != 1 { 3358 t.Errorf("expected substring %q exactly once in output.", sub) 3359 } 3360 } 3361 wantOnceOrMore := func(sub string) { 3362 if strings.Count(got, sub) == 0 { 3363 t.Errorf("expected substring %q at least once in output.", sub) 3364 } 3365 } 3366 wantOnce("Getting conn for dns-is-faked.golang:" + port) 3367 wantOnce("DNS start: {Host:dns-is-faked.golang}") 3368 wantOnce("DNS done: {Addrs:[{IP:" + ip + " Zone:}] Err:<nil> Coalesced:false}") 3369 wantOnce("got conn: {") 3370 wantOnceOrMore("Connecting to tcp " + addrStr) 3371 wantOnceOrMore("connected to tcp " + addrStr + " = <nil>") 3372 wantOnce("Reused:false WasIdle:false IdleTime:0s") 3373 wantOnce("first response byte") 3374 if h2 { 3375 wantOnce("tls handshake start") 3376 wantOnce("tls handshake done") 3377 } else { 3378 wantOnce("PutIdleConn = <nil>") 3379 } 3380 wantOnce("Wait100Continue") 3381 wantOnce("Got100Continue") 3382 wantOnce("WroteRequest: {Err:<nil>}") 3383 if strings.Contains(got, " to udp ") { 3384 t.Errorf("should not see UDP (DNS) connections") 3385 } 3386 if t.Failed() { 3387 t.Errorf("Output:\n%s", got) 3388 } 3389 } 3390 3391 func TestTransportEventTraceRealDNS(t *testing.T) { 3392 if testing.Short() && testenv.Builder() == "" { 3393 // Skip this test in short mode (the default for 3394 // all.bash), in case the user is using a shady/ISP 3395 // DNS server hijacking queries. 3396 // See issues 16732, 16716. 3397 // Our builders use 8.8.8.8, though, which correctly 3398 // returns NXDOMAIN, so still run this test there. 3399 t.Skip("skipping in short mode") 3400 } 3401 defer afterTest(t) 3402 tr := &Transport{} 3403 defer tr.CloseIdleConnections() 3404 c := &Client{Transport: tr} 3405 3406 var mu sync.Mutex // guards buf 3407 var buf bytes.Buffer 3408 logf := func(format string, args ...interface{}) { 3409 mu.Lock() 3410 defer mu.Unlock() 3411 fmt.Fprintf(&buf, format, args...) 3412 buf.WriteByte('\n') 3413 } 3414 3415 req, _ := NewRequest("GET", "http://dns-should-not-resolve.golang:80", nil) 3416 trace := &httptrace.ClientTrace{ 3417 DNSStart: func(e httptrace.DNSStartInfo) { logf("DNSStart: %+v", e) }, 3418 DNSDone: func(e httptrace.DNSDoneInfo) { logf("DNSDone: %+v", e) }, 3419 ConnectStart: func(network, addr string) { logf("ConnectStart: %s %s", network, addr) }, 3420 ConnectDone: func(network, addr string, err error) { logf("ConnectDone: %s %s %v", network, addr, err) }, 3421 } 3422 req = req.WithContext(httptrace.WithClientTrace(context.Background(), trace)) 3423 3424 resp, err := c.Do(req) 3425 if err == nil { 3426 resp.Body.Close() 3427 t.Fatal("expected error during DNS lookup") 3428 } 3429 3430 mu.Lock() 3431 got := buf.String() 3432 mu.Unlock() 3433 3434 wantSub := func(sub string) { 3435 if !strings.Contains(got, sub) { 3436 t.Errorf("expected substring %q in output.", sub) 3437 } 3438 } 3439 wantSub("DNSStart: {Host:dns-should-not-resolve.golang}") 3440 wantSub("DNSDone: {Addrs:[] Err:") 3441 if strings.Contains(got, "ConnectStart") || strings.Contains(got, "ConnectDone") { 3442 t.Errorf("should not see Connect events") 3443 } 3444 if t.Failed() { 3445 t.Errorf("Output:\n%s", got) 3446 } 3447 } 3448 3449 // Test the httptrace.TLSHandshake{Start,Done} hooks with a https http1 3450 // connections. The http2 test is done in TestTransportEventTrace_h2 3451 func TestTLSHandshakeTrace(t *testing.T) { 3452 defer afterTest(t) 3453 s := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) 3454 defer s.Close() 3455 3456 var mu sync.Mutex 3457 var start, done bool 3458 trace := &httptrace.ClientTrace{ 3459 TLSHandshakeStart: func() { 3460 mu.Lock() 3461 defer mu.Unlock() 3462 start = true 3463 }, 3464 TLSHandshakeDone: func(s tls.ConnectionState, err error) { 3465 mu.Lock() 3466 defer mu.Unlock() 3467 done = true 3468 if err != nil { 3469 t.Fatal("Expected error to be nil but was:", err) 3470 } 3471 }, 3472 } 3473 3474 tr := &Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} 3475 defer tr.CloseIdleConnections() 3476 c := &Client{Transport: tr} 3477 req, err := NewRequest("GET", s.URL, nil) 3478 if err != nil { 3479 t.Fatal("Unable to construct test request:", err) 3480 } 3481 req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) 3482 3483 r, err := c.Do(req) 3484 if err != nil { 3485 t.Fatal("Unexpected error making request:", err) 3486 } 3487 r.Body.Close() 3488 mu.Lock() 3489 defer mu.Unlock() 3490 if !start { 3491 t.Fatal("Expected TLSHandshakeStart to be called, but wasn't") 3492 } 3493 if !done { 3494 t.Fatal("Expected TLSHandshakeDone to be called, but wasnt't") 3495 } 3496 } 3497 3498 func TestTransportMaxIdleConns(t *testing.T) { 3499 defer afterTest(t) 3500 ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { 3501 // No body for convenience. 3502 })) 3503 defer ts.Close() 3504 tr := &Transport{ 3505 MaxIdleConns: 4, 3506 } 3507 defer tr.CloseIdleConnections() 3508 3509 ip, port, err := net.SplitHostPort(ts.Listener.Addr().String()) 3510 if err != nil { 3511 t.Fatal(err) 3512 } 3513 c := &Client{Transport: tr} 3514 ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { 3515 return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil 3516 }) 3517 3518 hitHost := func(n int) { 3519 req, _ := NewRequest("GET", fmt.Sprintf("http://host-%d.dns-is-faked.golang:"+port, n), nil) 3520 req = req.WithContext(ctx) 3521 res, err := c.Do(req) 3522 if err != nil { 3523 t.Fatal(err) 3524 } 3525 res.Body.Close() 3526 } 3527 for i := 0; i < 4; i++ { 3528 hitHost(i) 3529 } 3530 want := []string{ 3531 "|http|host-0.dns-is-faked.golang:" + port, 3532 "|http|host-1.dns-is-faked.golang:" + port, 3533 "|http|host-2.dns-is-faked.golang:" + port, 3534 "|http|host-3.dns-is-faked.golang:" + port, 3535 } 3536 if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { 3537 t.Fatalf("idle conn keys mismatch.\n got: %q\nwant: %q\n", got, want) 3538 } 3539 3540 // Now hitting the 5th host should kick out the first host: 3541 hitHost(4) 3542 want = []string{ 3543 "|http|host-1.dns-is-faked.golang:" + port, 3544 "|http|host-2.dns-is-faked.golang:" + port, 3545 "|http|host-3.dns-is-faked.golang:" + port, 3546 "|http|host-4.dns-is-faked.golang:" + port, 3547 } 3548 if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { 3549 t.Fatalf("idle conn keys mismatch after 5th host.\n got: %q\nwant: %q\n", got, want) 3550 } 3551 } 3552 3553 func TestTransportIdleConnTimeout_h1(t *testing.T) { testTransportIdleConnTimeout(t, h1Mode) } 3554 func TestTransportIdleConnTimeout_h2(t *testing.T) { testTransportIdleConnTimeout(t, h2Mode) } 3555 func testTransportIdleConnTimeout(t *testing.T, h2 bool) { 3556 if testing.Short() { 3557 t.Skip("skipping in short mode") 3558 } 3559 defer afterTest(t) 3560 3561 const timeout = 1 * time.Second 3562 3563 cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { 3564 // No body for convenience. 3565 })) 3566 defer cst.close() 3567 tr := cst.tr 3568 tr.IdleConnTimeout = timeout 3569 defer tr.CloseIdleConnections() 3570 c := &Client{Transport: tr} 3571 3572 idleConns := func() []string { 3573 if h2 { 3574 return tr.IdleConnStrsForTesting_h2() 3575 } else { 3576 return tr.IdleConnStrsForTesting() 3577 } 3578 } 3579 3580 var conn string 3581 doReq := func(n int) { 3582 req, _ := NewRequest("GET", cst.ts.URL, nil) 3583 req = req.WithContext(httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ 3584 PutIdleConn: func(err error) { 3585 if err != nil { 3586 t.Errorf("failed to keep idle conn: %v", err) 3587 } 3588 }, 3589 })) 3590 res, err := c.Do(req) 3591 if err != nil { 3592 t.Fatal(err) 3593 } 3594 res.Body.Close() 3595 conns := idleConns() 3596 if len(conns) != 1 { 3597 t.Fatalf("req %v: unexpected number of idle conns: %q", n, conns) 3598 } 3599 if conn == "" { 3600 conn = conns[0] 3601 } 3602 if conn != conns[0] { 3603 t.Fatalf("req %v: cached connection changed; expected the same one throughout the test", n) 3604 } 3605 } 3606 for i := 0; i < 3; i++ { 3607 doReq(i) 3608 time.Sleep(timeout / 2) 3609 } 3610 time.Sleep(timeout * 3 / 2) 3611 if got := idleConns(); len(got) != 0 { 3612 t.Errorf("idle conns = %q; want none", got) 3613 } 3614 } 3615 3616 // Issue 16208: Go 1.7 crashed after Transport.IdleConnTimeout if an 3617 // HTTP/2 connection was established but but its caller no longer 3618 // wanted it. (Assuming the connection cache was enabled, which it is 3619 // by default) 3620 // 3621 // This test reproduced the crash by setting the IdleConnTimeout low 3622 // (to make the test reasonable) and then making a request which is 3623 // canceled by the DialTLS hook, which then also waits to return the 3624 // real connection until after the RoundTrip saw the error. Then we 3625 // know the successful tls.Dial from DialTLS will need to go into the 3626 // idle pool. Then we give it a of time to explode. 3627 func TestIdleConnH2Crash(t *testing.T) { 3628 cst := newClientServerTest(t, h2Mode, HandlerFunc(func(w ResponseWriter, r *Request) { 3629 // nothing 3630 })) 3631 defer cst.close() 3632 3633 ctx, cancel := context.WithCancel(context.Background()) 3634 defer cancel() 3635 3636 gotErr := make(chan bool, 1) 3637 3638 cst.tr.IdleConnTimeout = 5 * time.Millisecond 3639 cst.tr.DialTLS = func(network, addr string) (net.Conn, error) { 3640 cancel() 3641 <-gotErr 3642 c, err := tls.Dial(network, addr, &tls.Config{ 3643 InsecureSkipVerify: true, 3644 NextProtos: []string{"h2"}, 3645 }) 3646 if err != nil { 3647 t.Error(err) 3648 return nil, err 3649 } 3650 if cs := c.ConnectionState(); cs.NegotiatedProtocol != "h2" { 3651 t.Errorf("protocol = %q; want %q", cs.NegotiatedProtocol, "h2") 3652 c.Close() 3653 return nil, errors.New("bogus") 3654 } 3655 return c, nil 3656 } 3657 3658 req, _ := NewRequest("GET", cst.ts.URL, nil) 3659 req = req.WithContext(ctx) 3660 res, err := cst.c.Do(req) 3661 if err == nil { 3662 res.Body.Close() 3663 t.Fatal("unexpected success") 3664 } 3665 gotErr <- true 3666 3667 // Wait for the explosion. 3668 time.Sleep(cst.tr.IdleConnTimeout * 10) 3669 } 3670 3671 type funcConn struct { 3672 net.Conn 3673 read func([]byte) (int, error) 3674 write func([]byte) (int, error) 3675 } 3676 3677 func (c funcConn) Read(p []byte) (int, error) { return c.read(p) } 3678 func (c funcConn) Write(p []byte) (int, error) { return c.write(p) } 3679 func (c funcConn) Close() error { return nil } 3680 3681 // Issue 16465: Transport.RoundTrip should return the raw net.Conn.Read error from Peek 3682 // back to the caller. 3683 func TestTransportReturnsPeekError(t *testing.T) { 3684 errValue := errors.New("specific error value") 3685 3686 wrote := make(chan struct{}) 3687 var wroteOnce sync.Once 3688 3689 tr := &Transport{ 3690 Dial: func(network, addr string) (net.Conn, error) { 3691 c := funcConn{ 3692 read: func([]byte) (int, error) { 3693 <-wrote 3694 return 0, errValue 3695 }, 3696 write: func(p []byte) (int, error) { 3697 wroteOnce.Do(func() { close(wrote) }) 3698 return len(p), nil 3699 }, 3700 } 3701 return c, nil 3702 }, 3703 } 3704 _, err := tr.RoundTrip(httptest.NewRequest("GET", "http://fake.tld/", nil)) 3705 if err != errValue { 3706 t.Errorf("error = %#v; want %v", err, errValue) 3707 } 3708 } 3709 3710 // Issue 13835: international domain names should work 3711 func TestTransportIDNA_h1(t *testing.T) { testTransportIDNA(t, h1Mode) } 3712 func TestTransportIDNA_h2(t *testing.T) { testTransportIDNA(t, h2Mode) } 3713 func testTransportIDNA(t *testing.T, h2 bool) { 3714 defer afterTest(t) 3715 3716 const uniDomain = "гофер.го" 3717 const punyDomain = "xn--c1ae0ajs.xn--c1aw" 3718 3719 var port string 3720 cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { 3721 want := punyDomain + ":" + port 3722 if r.Host != want { 3723 t.Errorf("Host header = %q; want %q", r.Host, want) 3724 } 3725 if h2 { 3726 if r.TLS == nil { 3727 t.Errorf("r.TLS == nil") 3728 } else if r.TLS.ServerName != punyDomain { 3729 t.Errorf("TLS.ServerName = %q; want %q", r.TLS.ServerName, punyDomain) 3730 } 3731 } 3732 w.Header().Set("Hit-Handler", "1") 3733 })) 3734 defer cst.close() 3735 3736 ip, port, err := net.SplitHostPort(cst.ts.Listener.Addr().String()) 3737 if err != nil { 3738 t.Fatal(err) 3739 } 3740 3741 // Install a fake DNS server. 3742 ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { 3743 if host != punyDomain { 3744 t.Errorf("got DNS host lookup for %q; want %q", host, punyDomain) 3745 return nil, nil 3746 } 3747 return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil 3748 }) 3749 3750 req, _ := NewRequest("GET", cst.scheme()+"://"+uniDomain+":"+port, nil) 3751 trace := &httptrace.ClientTrace{ 3752 GetConn: func(hostPort string) { 3753 want := net.JoinHostPort(punyDomain, port) 3754 if hostPort != want { 3755 t.Errorf("getting conn for %q; want %q", hostPort, want) 3756 } 3757 }, 3758 DNSStart: func(e httptrace.DNSStartInfo) { 3759 if e.Host != punyDomain { 3760 t.Errorf("DNSStart Host = %q; want %q", e.Host, punyDomain) 3761 } 3762 }, 3763 } 3764 req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) 3765 3766 res, err := cst.tr.RoundTrip(req) 3767 if err != nil { 3768 t.Fatal(err) 3769 } 3770 defer res.Body.Close() 3771 if res.Header.Get("Hit-Handler") != "1" { 3772 out, err := httputil.DumpResponse(res, true) 3773 if err != nil { 3774 t.Fatal(err) 3775 } 3776 t.Errorf("Response body wasn't from Handler. Got:\n%s\n", out) 3777 } 3778 } 3779 3780 var errFakeRoundTrip = errors.New("fake roundtrip") 3781 3782 type funcRoundTripper func() 3783 3784 func (fn funcRoundTripper) RoundTrip(*Request) (*Response, error) { 3785 fn() 3786 return nil, errFakeRoundTrip 3787 } 3788 3789 func wantBody(res *Response, err error, want string) error { 3790 if err != nil { 3791 return err 3792 } 3793 slurp, err := ioutil.ReadAll(res.Body) 3794 if err != nil { 3795 return fmt.Errorf("error reading body: %v", err) 3796 } 3797 if string(slurp) != want { 3798 return fmt.Errorf("body = %q; want %q", slurp, want) 3799 } 3800 if err := res.Body.Close(); err != nil { 3801 return fmt.Errorf("body Close = %v", err) 3802 } 3803 return nil 3804 } 3805 3806 func newLocalListener(t *testing.T) net.Listener { 3807 ln, err := net.Listen("tcp", "127.0.0.1:0") 3808 if err != nil { 3809 ln, err = net.Listen("tcp6", "[::1]:0") 3810 } 3811 if err != nil { 3812 t.Fatal(err) 3813 } 3814 return ln 3815 } 3816 3817 type countCloseReader struct { 3818 n *int 3819 io.Reader 3820 } 3821 3822 func (cr countCloseReader) Close() error { 3823 (*cr.n)++ 3824 return nil 3825 } 3826 3827 // rgz is a gzip quine that uncompresses to itself. 3828 var rgz = []byte{ 3829 0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 3830 0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 3831 0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0, 3832 0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 3833 0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 3834 0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60, 3835 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2, 3836 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00, 3837 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 3838 0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16, 3839 0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05, 3840 0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff, 3841 0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00, 3842 0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 3843 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 3844 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 3845 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 3846 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 3847 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 3848 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 3849 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 3850 0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff, 3851 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00, 3852 0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 3853 0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 3854 0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 3855 0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 3856 0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06, 3857 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00, 3858 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00, 3859 0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00, 3860 0x00, 0x00, 3861 }