github.com/hashicorp/go-getter/v2@v2.2.2/get_http_test.go (about) 1 package getter 2 3 import ( 4 "context" 5 "crypto/sha256" 6 "encoding/hex" 7 "errors" 8 "fmt" 9 "io/ioutil" 10 "net" 11 "net/http" 12 "net/http/httputil" 13 "net/url" 14 "os" 15 "path/filepath" 16 "strconv" 17 "strings" 18 "testing" 19 20 cleanhttp "github.com/hashicorp/go-cleanhttp" 21 testing_helper "github.com/hashicorp/go-getter/v2/helper/testing" 22 ) 23 24 func TestHttpGetter_impl(t *testing.T) { 25 var _ Getter = new(HttpGetter) 26 } 27 28 func TestHttpGetter_header(t *testing.T) { 29 ln := testHttpServer(t) 30 defer ln.Close() 31 ctx := context.Background() 32 33 g := new(HttpGetter) 34 dst := testing_helper.TempDir(t) 35 defer os.RemoveAll(dst) 36 37 var u url.URL 38 u.Scheme = "http" 39 u.Host = ln.Addr().String() 40 u.Path = "/header" 41 42 req := &Request{ 43 Dst: dst, 44 Src: u.String(), 45 u: &u, 46 GetMode: ModeDir, 47 } 48 49 // Get it, which should error because it uses the file protocol. 50 err := g.Get(ctx, req) 51 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol") { 52 t.Fatalf("unexpected error: %v", err) 53 } 54 // But, using a wrapper client with a file getter will work. 55 c := &Client{ 56 Getters: []Getter{ 57 g, 58 new(FileGetter), 59 }, 60 } 61 62 if _, err = c.Get(ctx, req); err != nil { 63 t.Fatalf("err: %s", err) 64 } 65 66 // Verify the main file exists 67 mainPath := filepath.Join(dst, "main.tf") 68 if _, err := os.Stat(mainPath); err != nil { 69 t.Fatalf("err: %s", err) 70 } 71 } 72 73 func TestHttpGetter_requestHeader(t *testing.T) { 74 ln := testHttpServer(t) 75 defer ln.Close() 76 ctx := context.Background() 77 78 g := new(HttpGetter) 79 g.Header = make(http.Header) 80 g.Header.Add("X-Foobar", "foobar") 81 dst := testing_helper.TempDir(t) 82 defer os.RemoveAll(dst) 83 84 var u url.URL 85 u.Scheme = "http" 86 u.Host = ln.Addr().String() 87 u.Path = "/expect-header" 88 u.RawQuery = "expected=X-Foobar" 89 90 req := &Request{ 91 Dst: dst, 92 u: &u, 93 } 94 95 // Get it! 96 if err := g.GetFile(ctx, req); err != nil { 97 t.Fatalf("err: %s", err) 98 } 99 100 // Verify the main file exists 101 if _, err := os.Stat(dst); err != nil { 102 t.Fatalf("err: %s", err) 103 } 104 testing_helper.AssertContents(t, dst, "Hello\n") 105 } 106 107 func TestHttpGetter_meta(t *testing.T) { 108 ln := testHttpServer(t) 109 defer ln.Close() 110 ctx := context.Background() 111 112 g := new(HttpGetter) 113 dst := testing_helper.TempDir(t) 114 defer os.RemoveAll(dst) 115 116 var u url.URL 117 u.Scheme = "http" 118 u.Host = ln.Addr().String() 119 u.Path = "/meta" 120 121 req := &Request{ 122 Dst: dst, 123 Src: u.String(), 124 u: &u, 125 GetMode: ModeDir, 126 } 127 128 // Get it, which should error because it uses the file protocol. 129 err := g.Get(ctx, req) 130 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol:") { 131 t.Fatalf("unexpected error: %v", err) 132 } 133 // But, using a wrapper client with a file getter will work. 134 c := &Client{ 135 Getters: []Getter{ 136 g, 137 new(FileGetter), 138 }, 139 } 140 141 if _, err = c.Get(ctx, req); err != nil { 142 t.Fatalf("err: %s", err) 143 } 144 145 // Verify the main file exists 146 mainPath := filepath.Join(dst, "main.tf") 147 if _, err := os.Stat(mainPath); err != nil { 148 t.Fatalf("err: %s", err) 149 } 150 } 151 152 func TestHttpGetter_metaSubdir(t *testing.T) { 153 ln := testHttpServer(t) 154 defer ln.Close() 155 ctx := context.Background() 156 157 g := new(HttpGetter) 158 dst := testing_helper.TempDir(t) 159 defer os.RemoveAll(dst) 160 161 var u url.URL 162 u.Scheme = "http" 163 u.Host = ln.Addr().String() 164 u.Path = "/meta-subdir" 165 166 req := &Request{ 167 Dst: dst, 168 Src: u.String(), 169 u: &u, 170 GetMode: ModeDir, 171 } 172 173 // Get it, which should error because it uses the file protocol. 174 err := g.Get(ctx, req) 175 if !strings.Contains(err.Error(), "error downloading") { 176 t.Fatalf("unexpected error: %v", err) 177 } 178 // But, using a wrapper client with a file getter will work. 179 c := &Client{ 180 Getters: []Getter{ 181 g, 182 new(FileGetter), 183 }, 184 } 185 186 if _, err = c.Get(ctx, req); err != nil { 187 t.Fatalf("err: %s", err) 188 } 189 190 // Verify the main file exists 191 mainPath := filepath.Join(dst, "sub.tf") 192 if _, err := os.Stat(mainPath); err != nil { 193 t.Fatalf("err: %s", err) 194 } 195 } 196 197 func TestHttpGetter_metaSubdirGlob(t *testing.T) { 198 ln := testHttpServer(t) 199 defer ln.Close() 200 ctx := context.Background() 201 202 g := new(HttpGetter) 203 dst := testing_helper.TempDir(t) 204 defer os.RemoveAll(dst) 205 206 var u url.URL 207 u.Scheme = "http" 208 u.Host = ln.Addr().String() 209 u.Path = "/meta-subdir-glob" 210 211 req := &Request{ 212 Dst: dst, 213 Src: u.String(), 214 u: &u, 215 GetMode: ModeDir, 216 } 217 218 // Get it, which should error because it uses the file protocol. 219 err := g.Get(ctx, req) 220 if !strings.Contains(err.Error(), "error downloading") { 221 t.Fatalf("unexpected error: %v", err) 222 } 223 // But, using a wrapper client with a file getter will work. 224 c := &Client{ 225 Getters: []Getter{ 226 g, 227 new(FileGetter), 228 }, 229 } 230 231 if _, err = c.Get(ctx, req); err != nil { 232 t.Fatalf("err: %s", err) 233 } 234 235 // Verify the main file exists 236 mainPath := filepath.Join(dst, "sub.tf") 237 if _, err := os.Stat(mainPath); err != nil { 238 t.Fatalf("err: %s", err) 239 } 240 } 241 242 func TestHttpGetter_none(t *testing.T) { 243 ln := testHttpServer(t) 244 defer ln.Close() 245 ctx := context.Background() 246 247 g := new(HttpGetter) 248 dst := testing_helper.TempDir(t) 249 defer os.RemoveAll(dst) 250 251 var u url.URL 252 u.Scheme = "http" 253 u.Host = ln.Addr().String() 254 u.Path = "/none" 255 256 req := &Request{ 257 Dst: dst, 258 u: &u, 259 } 260 261 // Get it! 262 if err := g.Get(ctx, req); err == nil { 263 t.Fatal("should error") 264 } 265 } 266 267 func TestHttpGetter_resume(t *testing.T) { 268 load := []byte(testHttpMetaStr) 269 sha := sha256.New() 270 if n, err := sha.Write(load); n != len(load) || err != nil { 271 t.Fatalf("sha write failed: %d, %s", n, err) 272 } 273 checksum := hex.EncodeToString(sha.Sum(nil)) 274 downloadFrom := len(load) / 2 275 276 ln := testHttpServer(t) 277 defer ln.Close() 278 279 dst := testing_helper.TempDir(t) 280 defer os.RemoveAll(dst) 281 282 dst = filepath.Join(dst, "..", "range") 283 f, err := os.Create(dst) 284 if err != nil { 285 t.Fatalf("create: %v", err) 286 } 287 if n, err := f.Write(load[:downloadFrom]); n != downloadFrom || err != nil { 288 t.Fatalf("partial file write failed: %d, %s", n, err) 289 } 290 if err := f.Close(); err != nil { 291 t.Fatalf("close failed: %s", err) 292 } 293 294 u := url.URL{ 295 Scheme: "http", 296 Host: ln.Addr().String(), 297 Path: "/range", 298 RawQuery: "checksum=" + checksum, 299 } 300 t.Logf("url: %s", u.String()) 301 ctx := context.Background() 302 303 // Finish getting it! 304 if _, err := GetFile(ctx, dst, u.String()); err != nil { 305 t.Fatalf("finishing download should not error: %v", err) 306 } 307 308 b, err := ioutil.ReadFile(dst) 309 if err != nil { 310 t.Fatalf("readfile failed: %v", err) 311 } 312 313 if string(b) != string(load) { 314 t.Fatalf("file differs: got:\n%s\n expected:\n%s\n", string(b), string(load)) 315 } 316 317 // Get it again 318 if _, err := GetFile(ctx, dst, u.String()); err != nil { 319 t.Fatalf("should not error: %v", err) 320 } 321 } 322 323 // The server may support Byte-Range, but has no size for the requested object 324 func TestHttpGetter_resumeNoRange(t *testing.T) { 325 load := []byte(testHttpMetaStr) 326 sha := sha256.New() 327 if n, err := sha.Write(load); n != len(load) || err != nil { 328 t.Fatalf("sha write failed: %d, %s", n, err) 329 } 330 checksum := hex.EncodeToString(sha.Sum(nil)) 331 downloadFrom := len(load) / 2 332 333 ln := testHttpServer(t) 334 defer ln.Close() 335 336 dst := testing_helper.TempDir(t) 337 defer os.RemoveAll(dst) 338 339 dst = filepath.Join(dst, "..", "range") 340 f, err := os.Create(dst) 341 if err != nil { 342 t.Fatalf("create: %v", err) 343 } 344 if n, err := f.Write(load[:downloadFrom]); n != downloadFrom || err != nil { 345 t.Fatalf("partial file write failed: %d, %s", n, err) 346 } 347 if err := f.Close(); err != nil { 348 t.Fatalf("close failed: %s", err) 349 } 350 351 u := url.URL{ 352 Scheme: "http", 353 Host: ln.Addr().String(), 354 Path: "/no-range", 355 RawQuery: "checksum=" + checksum, 356 } 357 t.Logf("url: %s", u.String()) 358 ctx := context.Background() 359 360 // Finish getting it! 361 if _, err := GetFile(ctx, dst, u.String()); err != nil { 362 t.Fatalf("finishing download should not error: %v", err) 363 } 364 365 b, err := ioutil.ReadFile(dst) 366 if err != nil { 367 t.Fatalf("readfile failed: %v", err) 368 } 369 370 if string(b) != string(load) { 371 t.Fatalf("file differs: got:\n%s\n expected:\n%s\n", string(b), string(load)) 372 } 373 } 374 375 func TestHttpGetter_file(t *testing.T) { 376 ln := testHttpServer(t) 377 defer ln.Close() 378 ctx := context.Background() 379 380 g := new(HttpGetter) 381 dst := testing_helper.TempTestFile(t) 382 defer os.RemoveAll(filepath.Dir(dst)) 383 384 var u url.URL 385 u.Scheme = "http" 386 u.Host = ln.Addr().String() 387 u.Path = "/file" 388 389 req := &Request{ 390 Dst: dst, 391 u: &u, 392 } 393 394 // Get it! 395 if err := g.GetFile(ctx, req); err != nil { 396 t.Fatalf("err: %s", err) 397 } 398 399 // Verify the main file exists 400 if _, err := os.Stat(dst); err != nil { 401 t.Fatalf("err: %s", err) 402 } 403 testing_helper.AssertContents(t, dst, "Hello\n") 404 } 405 406 func TestHttpGetter_auth(t *testing.T) { 407 ln := testHttpServer(t) 408 defer ln.Close() 409 ctx := context.Background() 410 411 g := new(HttpGetter) 412 dst := testing_helper.TempDir(t) 413 defer os.RemoveAll(dst) 414 415 var u url.URL 416 u.Scheme = "http" 417 u.Host = ln.Addr().String() 418 u.Path = "/meta-auth" 419 u.User = url.UserPassword("foo", "bar") 420 421 req := &Request{ 422 Dst: dst, 423 Src: u.String(), 424 u: &u, 425 GetMode: ModeDir, 426 } 427 428 // Get it, which should error because it uses the file protocol. 429 err := g.Get(ctx, req) 430 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol:") { 431 t.Fatalf("unexpected error: %v", err) 432 } 433 // But, using a wrapper client with a file getter will work. 434 c := &Client{ 435 Getters: []Getter{ 436 g, 437 new(FileGetter), 438 }, 439 } 440 441 if _, err = c.Get(ctx, req); err != nil { 442 t.Fatalf("err: %s", err) 443 } 444 445 // Verify the main file exists 446 mainPath := filepath.Join(dst, "main.tf") 447 if _, err := os.Stat(mainPath); err != nil { 448 t.Fatalf("err: %s", err) 449 } 450 } 451 452 func TestHttpGetter_authNetrc(t *testing.T) { 453 ln := testHttpServer(t) 454 defer ln.Close() 455 ctx := context.Background() 456 457 g := new(HttpGetter) 458 dst := testing_helper.TempDir(t) 459 defer os.RemoveAll(dst) 460 461 var u url.URL 462 u.Scheme = "http" 463 u.Host = ln.Addr().String() 464 u.Path = "/meta" 465 466 // Write the netrc file 467 path, closer := testing_helper.TempFileWithContent(t, fmt.Sprintf(testHttpNetrc, ln.Addr().String())) 468 defer closer() 469 defer tempEnv(t, "NETRC", path)() 470 471 req := &Request{ 472 Dst: dst, 473 Src: u.String(), 474 u: &u, 475 GetMode: ModeDir, 476 } 477 478 // Get it, which should error because it uses the file protocol. 479 err := g.Get(ctx, req) 480 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol:") { 481 t.Fatalf("unexpected error: %v", err) 482 } 483 // But, using a wrapper client with a file getter will work. 484 c := &Client{ 485 Getters: []Getter{ 486 g, 487 new(FileGetter), 488 }, 489 } 490 491 if _, err = c.Get(ctx, req); err != nil { 492 t.Fatalf("err: %s", err) 493 } 494 495 // Verify the main file exists 496 mainPath := filepath.Join(dst, "main.tf") 497 if _, err := os.Stat(mainPath); err != nil { 498 t.Fatalf("err: %s", err) 499 } 500 } 501 502 // test round tripper that only returns an error 503 type errRoundTripper struct{} 504 505 func (errRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { 506 return nil, errors.New("test round tripper") 507 } 508 509 // verify that the default httpClient no longer comes from http.DefaultClient 510 func TestHttpGetter_cleanhttp(t *testing.T) { 511 ln := testHttpServer(t) 512 defer ln.Close() 513 514 // break the default http client 515 http.DefaultClient.Transport = errRoundTripper{} 516 defer func() { 517 http.DefaultClient.Transport = http.DefaultTransport 518 }() 519 ctx := context.Background() 520 521 g := new(HttpGetter) 522 dst := testing_helper.TempDir(t) 523 defer os.RemoveAll(dst) 524 525 var u url.URL 526 u.Scheme = "http" 527 u.Host = ln.Addr().String() 528 u.Path = "/header" 529 530 req := &Request{ 531 Dst: dst, 532 Src: u.String(), 533 u: &u, 534 GetMode: ModeDir, 535 } 536 537 // Get it, which should error because it uses the file protocol. 538 err := g.Get(ctx, req) 539 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol:") { 540 t.Fatalf("unexpected error: %v", err) 541 } 542 // But, using a wrapper client with a file getter will work. 543 c := &Client{ 544 Getters: []Getter{ 545 g, 546 new(FileGetter), 547 }, 548 } 549 550 if _, err = c.Get(ctx, req); err != nil { 551 t.Fatalf("err: %s", err) 552 } 553 554 } 555 556 func TestHttpGetter__RespectsContextCanceled(t *testing.T) { 557 ctx, cancel := context.WithCancel(context.Background()) 558 cancel() // cancel immediately 559 560 ln := testHttpServer(t) 561 562 var u url.URL 563 u.Scheme = "http" 564 u.Host = ln.Addr().String() 565 u.Path = "/file" 566 567 dst := testing_helper.TempDir(t) 568 defer os.RemoveAll(dst) 569 570 rt := hookableHTTPRoundTripper{ 571 before: func(req *http.Request) { 572 err := req.Context().Err() 573 if !errors.Is(err, context.Canceled) { 574 t.Fatalf("Expected http.Request with canceled.Context, got: %v", err) 575 } 576 }, 577 RoundTripper: http.DefaultTransport, 578 } 579 580 g := new(HttpGetter) 581 g.Client = &http.Client{ 582 Transport: &rt, 583 } 584 585 req := Request{ 586 Dst: dst, 587 u: &u, 588 } 589 err := g.Get(ctx, &req) 590 if !errors.Is(err, context.Canceled) { 591 t.Fatalf("expected context.Canceled, got: %v", err) 592 } 593 } 594 595 func TestHttpGetter__XTerraformGetLimit(t *testing.T) { 596 ctx, cancel := context.WithCancel(context.Background()) 597 defer cancel() 598 599 ln := testHttpServerWithXTerraformGetLoop(t) 600 601 var u url.URL 602 u.Scheme = "http" 603 u.Host = ln.Addr().String() 604 u.Path = "/loop" 605 606 dst := testing_helper.TempDir(t) 607 defer os.RemoveAll(dst) 608 609 g := new(HttpGetter) 610 g.XTerraformGetLimit = 10 611 g.Client = &http.Client{} 612 613 req := Request{ 614 Dst: dst, 615 u: &u, 616 GetMode: ModeDir, 617 } 618 619 err := g.Get(ctx, &req) 620 if !strings.Contains(err.Error(), "too many X-Terraform-Get redirects") { 621 t.Fatalf("too many X-Terraform-Get redirects, got: %v", err) 622 } 623 } 624 625 func TestHttpGetter__XTerraformGetDisabled(t *testing.T) { 626 ctx, cancel := context.WithCancel(context.Background()) 627 defer cancel() 628 629 ln := testHttpServerWithXTerraformGetLoop(t) 630 631 var u url.URL 632 u.Scheme = "http" 633 u.Host = ln.Addr().String() 634 u.Path = "/loop" 635 dst := testing_helper.TempDir(t) 636 637 g := new(HttpGetter) 638 g.XTerraformGetDisabled = true 639 g.Client = &http.Client{} 640 641 req := Request{ 642 Dst: dst, 643 u: &u, 644 GetMode: ModeDir, 645 } 646 647 err := g.Get(ctx, &req) 648 if err != nil { 649 t.Fatalf("unexpected error: %v", err) 650 } 651 } 652 func TestHttpGetter__XTerraformGetProxyBypass(t *testing.T) { 653 ctx, cancel := context.WithCancel(context.Background()) 654 defer cancel() 655 656 ln := testHttpServerWithXTerraformGetProxyBypass(t) 657 658 proxyLn := testHttpServerProxy(t, ln.Addr().String()) 659 660 t.Logf("starting malicious server on: %v", ln.Addr().String()) 661 t.Logf("starting proxy on: %v", proxyLn.Addr().String()) 662 663 var u url.URL 664 u.Scheme = "http" 665 u.Host = ln.Addr().String() 666 u.Path = "/start" 667 dst := testing_helper.TempDir(t) 668 669 proxy, err := url.Parse(fmt.Sprintf("http://%s/", proxyLn.Addr().String())) 670 if err != nil { 671 t.Fatalf("failed to parse proxy URL: %v", err) 672 } 673 674 transport := cleanhttp.DefaultTransport() 675 transport.Proxy = http.ProxyURL(proxy) 676 677 g := new(HttpGetter) 678 g.XTerraformGetLimit = 10 679 g.Client = &http.Client{ 680 Transport: transport, 681 } 682 683 client := &Client{ 684 Getters: []Getter{g}, 685 } 686 687 req := Request{ 688 Dst: dst, 689 Src: u.String(), 690 } 691 692 _, err = client.Get(ctx, &req) 693 if err != nil { 694 t.Logf("client get error: %v", err) 695 } 696 } 697 698 func TestHttpGetter__XTerraformGetConfiguredGettersBypass(t *testing.T) { 699 tc := []struct { 700 name string 701 configuredGetters []Getter 702 errExpected bool 703 }{ 704 {name: "configured getter for git protocol switch", configuredGetters: []Getter{new(GitGetter)}, errExpected: false}, 705 {name: "configured getter for multiple protocol switch", configuredGetters: []Getter{new(GitGetter), new(HgGetter), new(FileGetter)}, errExpected: false}, 706 {name: "configured getter for file protocol switch", configuredGetters: []Getter{new(FileGetter)}, errExpected: true}, 707 } 708 709 for _, tt := range tc { 710 tt := tt 711 t.Run(tt.name, func(t *testing.T) { 712 ctx, cancel := context.WithCancel(context.Background()) 713 defer cancel() 714 715 ln := testHttpServerWithXTerraformGetConfiguredGettersBypass(t) 716 717 var u url.URL 718 u.Scheme = "http" 719 u.Host = ln.Addr().String() 720 u.Path = "/start" 721 722 dst := testing_helper.TempDir(t) 723 724 rt := hookableHTTPRoundTripper{ 725 before: func(req *http.Request) { 726 t.Logf("making request") 727 }, 728 RoundTripper: http.DefaultTransport, 729 } 730 731 g := new(HttpGetter) 732 g.XTerraformGetLimit = 10 733 g.Client = &http.Client{ 734 Transport: &rt, 735 } 736 737 client := &Client{ 738 Getters: []Getter{g}, 739 } 740 client.Getters = append(client.Getters, tt.configuredGetters...) 741 742 t.Logf("%v", u.String()) 743 744 req := Request{ 745 Dst: dst, 746 Src: u.String(), 747 GetMode: ModeDir, 748 } 749 750 _, err := client.Get(ctx, &req) 751 // For configured getters that support git, the git repository doesn't exist so error will not be nil. 752 // If we get a nil error when we expect one other than the git error git exited with -1 we should fail. 753 if tt.errExpected && err == nil { 754 t.Fatalf("error expected") 755 } 756 // We only care about the error messages that indicate that we can download the git header URL 757 if tt.errExpected && err != nil { 758 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol:") { 759 t.Fatalf("expected no getter available for X-Terraform-Get source protocol:, got: %v", err) 760 } 761 } 762 }) 763 } 764 } 765 766 func TestHttpGetter__endless_body(t *testing.T) { 767 ctx, cancel := context.WithCancel(context.Background()) 768 defer cancel() 769 770 ln := testHttpServerWithEndlessBody(t) 771 772 var u url.URL 773 u.Scheme = "http" 774 u.Host = ln.Addr().String() 775 u.Path = "/" 776 dst := testing_helper.TempDir(t) 777 778 g := new(HttpGetter) 779 g.MaxBytes = 10 780 g.DoNotCheckHeadFirst = true 781 782 client := &Client{ 783 Getters: []Getter{g}, 784 } 785 786 t.Logf("%v", u.String()) 787 788 req := Request{ 789 Dst: dst, 790 Src: u.String(), 791 GetMode: ModeFile, 792 } 793 794 _, err := client.Get(ctx, &req) 795 if err != nil { 796 t.Fatalf("unexpected error: %v", err) 797 } 798 } 799 800 func TestHttpGetter_subdirLink(t *testing.T) { 801 ctx, cancel := context.WithCancel(context.Background()) 802 defer cancel() 803 804 ln := testHttpServerSubDir(t) 805 defer ln.Close() 806 807 dst, err := ioutil.TempDir("", "tf") 808 if err != nil { 809 t.Fatalf("err: %s", err) 810 } 811 812 t.Logf("dst: %q", dst) 813 814 var u url.URL 815 u.Scheme = "http" 816 u.Host = ln.Addr().String() 817 u.Path = "/regular-subdir//meta-subdir" 818 819 g := new(HttpGetter) 820 client := &Client{ 821 Getters: []Getter{g}, 822 } 823 824 t.Logf("url: %q", u.String()) 825 826 req := Request{ 827 Dst: dst, 828 Src: u.String(), 829 GetMode: ModeAny, 830 } 831 832 _, err = client.Get(ctx, &req) 833 if err != nil { 834 t.Fatalf("get err: %v", err) 835 } 836 } 837 838 func testHttpServerWithXTerraformGetLoop(t *testing.T) net.Listener { 839 t.Helper() 840 841 ln, err := net.Listen("tcp", "127.0.0.1:0") 842 if err != nil { 843 t.Fatalf("err: %s", err) 844 } 845 846 header := fmt.Sprintf("http://%v:%v", ln.Addr().String(), "/loop") 847 848 mux := http.NewServeMux() 849 mux.HandleFunc("/loop", func(w http.ResponseWriter, r *http.Request) { 850 w.Header().Set("X-Terraform-Get", header) 851 t.Logf("serving loop") 852 }) 853 854 var server http.Server 855 server.Handler = mux 856 go server.Serve(ln) 857 858 return ln 859 } 860 861 func testHttpServerWithXTerraformGetProxyBypass(t *testing.T) net.Listener { 862 t.Helper() 863 864 ln, err := net.Listen("tcp", "127.0.0.1:0") 865 if err != nil { 866 t.Fatalf("err: %s", err) 867 } 868 869 header := fmt.Sprintf("http://%v/bypass", ln.Addr().String()) 870 871 mux := http.NewServeMux() 872 mux.HandleFunc("/start/start", func(w http.ResponseWriter, r *http.Request) { 873 w.Header().Set("X-Terraform-Get", header) 874 t.Logf("serving start") 875 }) 876 877 mux.HandleFunc("/bypass", func(w http.ResponseWriter, r *http.Request) { 878 t.Fail() 879 t.Logf("bypassed proxy") 880 }) 881 882 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 883 t.Logf("serving HTTP server path: %v", r.URL.Path) 884 }) 885 886 var server http.Server 887 server.Handler = mux 888 go server.Serve(ln) 889 890 return ln 891 } 892 893 func testHttpServerWithXTerraformGetConfiguredGettersBypass(t *testing.T) net.Listener { 894 t.Helper() 895 896 ln, err := net.Listen("tcp", "127.0.0.1:0") 897 if err != nil { 898 t.Fatalf("err: %s", err) 899 } 900 901 header := fmt.Sprintf("git::http://%v/some/repository.git", ln.Addr().String()) 902 903 mux := http.NewServeMux() 904 mux.HandleFunc("/start", func(w http.ResponseWriter, r *http.Request) { 905 w.Header().Set("X-Terraform-Get", header) 906 t.Logf("serving start") 907 }) 908 909 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 910 t.Logf("serving git HTTP server path: %v", r.URL.Path) 911 }) 912 913 var server http.Server 914 server.Handler = mux 915 go server.Serve(ln) 916 917 return ln 918 } 919 920 func TestHttpGetter_XTerraformWithClientFromContext(t *testing.T) { 921 tc := []struct { 922 name string 923 client *Client 924 errExpected bool 925 }{ 926 { 927 name: "default getters", 928 client: &Client{ 929 Getters: Getters, 930 }, 931 errExpected: false, 932 }, 933 { 934 name: "client configured with needed getters", 935 client: &Client{ 936 Getters: []Getter{ 937 new(HttpGetter), 938 new(FileGetter), 939 }, 940 }, 941 errExpected: false, 942 }, 943 { 944 name: "nil client", 945 errExpected: true, 946 }, 947 } 948 949 for _, tt := range tc { 950 tt := tt 951 t.Run(tt.name, func(t *testing.T) { 952 ln := testHttpServer(t) 953 defer ln.Close() 954 ctx := context.Background() 955 956 g := new(HttpGetter) 957 dst := testing_helper.TempDir(t) 958 defer os.RemoveAll(dst) 959 960 var u url.URL 961 u.Scheme = "http" 962 u.Host = ln.Addr().String() 963 u.Path = "/header" 964 965 req := &Request{ 966 Dst: dst, 967 Src: u.String(), 968 u: &u, 969 GetMode: ModeDir, 970 } 971 972 // Using a client stored in the ctx with a file getter should work 973 ctx = NewContextWithClient(ctx, tt.client) 974 975 err := g.Get(ctx, req) 976 if tt.errExpected && err == nil { 977 t.Fatalf("error expected") 978 } 979 980 if err != nil { 981 if !strings.Contains(err.Error(), "no getter available for X-Terraform-Get source protocol:") { 982 t.Fatalf("expected no getter available for X-Terraform-Get source protocol:, got: %v", err) 983 } 984 return 985 } 986 987 // Verify the main file exists 988 mainPath := filepath.Join(dst, "main.tf") 989 if _, err := os.Stat(mainPath); err != nil { 990 t.Fatalf("err: %s", err) 991 } 992 }) 993 } 994 } 995 996 func testHttpServerProxy(t *testing.T, upstreamHost string) net.Listener { 997 t.Helper() 998 999 ln, err := net.Listen("tcp", "127.0.0.1:0") 1000 if err != nil { 1001 t.Fatalf("err: %s", err) 1002 } 1003 1004 mux := http.NewServeMux() 1005 1006 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 1007 t.Logf("serving proxy: %v: %#+v", r.URL.Path, r.Header) 1008 // create the reverse proxy 1009 proxy := httputil.NewSingleHostReverseProxy(r.URL) 1010 // Note that ServeHttp is non blocking & uses a go routine under the hood 1011 proxy.ServeHTTP(w, r) 1012 }) 1013 1014 var server http.Server 1015 server.Handler = mux 1016 go server.Serve(ln) 1017 1018 return ln 1019 } 1020 1021 func testHttpServer(t *testing.T) net.Listener { 1022 ln, err := net.Listen("tcp", "127.0.0.1:0") 1023 if err != nil { 1024 t.Fatalf("err: %s", err) 1025 } 1026 1027 mux := http.NewServeMux() 1028 mux.HandleFunc("/expect-header", testHttpHandlerExpectHeader) 1029 mux.HandleFunc("/file", testHttpHandlerFile) 1030 mux.HandleFunc("/header", testHttpHandlerHeader) 1031 mux.HandleFunc("/meta", testHttpHandlerMeta) 1032 mux.HandleFunc("/meta-auth", testHttpHandlerMetaAuth) 1033 mux.HandleFunc("/meta-subdir", testHttpHandlerMetaSubdir) 1034 mux.HandleFunc("/meta-subdir-glob", testHttpHandlerMetaSubdirGlob) 1035 mux.HandleFunc("/range", testHttpHandlerRange) 1036 mux.HandleFunc("/no-range", testHttpHandlerNoRange) 1037 1038 var server http.Server 1039 server.Handler = mux 1040 go server.Serve(ln) 1041 1042 return ln 1043 } 1044 1045 func testHttpServerWithEndlessBody(t *testing.T) net.Listener { 1046 t.Helper() 1047 1048 ln, err := net.Listen("tcp", "127.0.0.1:0") 1049 if err != nil { 1050 t.Fatalf("err: %s", err) 1051 } 1052 1053 mux := http.NewServeMux() 1054 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 1055 w.WriteHeader(http.StatusOK) 1056 for { 1057 w.Write([]byte(".\n")) 1058 } 1059 }) 1060 1061 var server http.Server 1062 server.Handler = mux 1063 go server.Serve(ln) 1064 1065 return ln 1066 } 1067 1068 func testHttpHandlerExpectHeader(w http.ResponseWriter, r *http.Request) { 1069 if expected, ok := r.URL.Query()["expected"]; ok { 1070 if r.Header.Get(expected[0]) != "" { 1071 w.Write([]byte("Hello\n")) 1072 return 1073 } 1074 } 1075 1076 w.WriteHeader(400) 1077 } 1078 1079 func testHttpHandlerFile(w http.ResponseWriter, r *http.Request) { 1080 w.Write([]byte("Hello\n")) 1081 } 1082 1083 func testHttpHandlerHeader(w http.ResponseWriter, r *http.Request) { 1084 w.Header().Add("X-Terraform-Get", testModuleURL("basic").String()) 1085 w.WriteHeader(200) 1086 } 1087 1088 func testHttpHandlerMeta(w http.ResponseWriter, r *http.Request) { 1089 w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String()))) 1090 } 1091 1092 func testHttpHandlerMetaAuth(w http.ResponseWriter, r *http.Request) { 1093 user, pass, ok := r.BasicAuth() 1094 if !ok { 1095 w.WriteHeader(401) 1096 return 1097 } 1098 1099 if user != "foo" || pass != "bar" { 1100 w.WriteHeader(401) 1101 return 1102 } 1103 1104 w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String()))) 1105 } 1106 1107 func testHttpHandlerMetaSubdir(w http.ResponseWriter, r *http.Request) { 1108 w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//subdir").String()))) 1109 } 1110 1111 func testHttpHandlerMetaSubdirGlob(w http.ResponseWriter, r *http.Request) { 1112 w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//sub*").String()))) 1113 } 1114 1115 func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) { 1116 w.Write([]byte(testHttpNoneStr)) 1117 } 1118 1119 func testHttpHandlerRange(w http.ResponseWriter, r *http.Request) { 1120 load := []byte(testHttpMetaStr) 1121 switch r.Method { 1122 case "HEAD": 1123 w.Header().Add("accept-ranges", "bytes") 1124 w.Header().Add("content-length", strconv.Itoa(len(load))) 1125 default: 1126 // request should have header "Range: bytes=0-1023" 1127 // or "Range: bytes=123-" 1128 rangeHeaderValue := strings.Split(r.Header.Get("Range"), "=")[1] 1129 rng, _ := strconv.Atoi(strings.Split(rangeHeaderValue, "-")[0]) 1130 if rng < 1 || rng > len(load) { 1131 http.Error(w, "", http.StatusBadRequest) 1132 } 1133 w.Write(load[rng:]) 1134 } 1135 } 1136 1137 func testHttpHandlerNoRange(w http.ResponseWriter, r *http.Request) { 1138 load := []byte(testHttpMetaStr) 1139 switch r.Method { 1140 case "HEAD": 1141 // we support range, but the object size isn't known 1142 w.Header().Add("accept-ranges", "bytes") 1143 default: 1144 if r.Header.Get("Range") != "" { 1145 http.Error(w, "range not supported", http.StatusBadRequest) 1146 } 1147 w.Write(load) 1148 } 1149 } 1150 1151 func testHttpServerSubDir(t *testing.T) net.Listener { 1152 t.Helper() 1153 1154 ln, err := net.Listen("tcp", "127.0.0.1:0") 1155 if err != nil { 1156 t.Fatalf("err: %s", err) 1157 } 1158 1159 mux := http.NewServeMux() 1160 mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 1161 switch r.Method { 1162 case http.MethodGet: 1163 t.Logf("serving: %v: %v: %#+[1]v", r.Method, r.URL.String(), r.Header) 1164 } 1165 }) 1166 1167 var server http.Server 1168 server.Handler = mux 1169 go server.Serve(ln) 1170 1171 return ln 1172 } 1173 1174 const testHttpMetaStr = ` 1175 <html> 1176 <head> 1177 <meta name="terraform-get" content="%s"> 1178 </head> 1179 </html> 1180 ` 1181 1182 const testHttpNoneStr = ` 1183 <html> 1184 <head> 1185 </head> 1186 </html> 1187 ` 1188 1189 const testHttpNetrc = ` 1190 machine %s 1191 login foo 1192 password bar 1193 ` 1194 1195 type hookableHTTPRoundTripper struct { 1196 before func(req *http.Request) 1197 http.RoundTripper 1198 } 1199 1200 func (m *hookableHTTPRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 1201 if m.before != nil { 1202 m.before(req) 1203 } 1204 return m.RoundTripper.RoundTrip(req) 1205 }