github.com/peterdeka/grab@v2.0.0+incompatible/client_test.go (about) 1 package grab 2 3 import ( 4 "context" 5 "crypto/md5" 6 "crypto/sha1" 7 "crypto/sha256" 8 "crypto/sha512" 9 "encoding/hex" 10 "errors" 11 "fmt" 12 "hash" 13 "math/rand" 14 "net/http" 15 "os" 16 "strings" 17 "testing" 18 "time" 19 ) 20 21 // TestFilenameResolutions tests that the destination filename for Requests can 22 // be determined correctly, using an explicitly requested path, 23 // Content-Disposition headers or a URL path - with or without an existing 24 // target directory. 25 func TestFilenameResolution(t *testing.T) { 26 testCases := []struct { 27 Name string 28 Filename string 29 URL string 30 Expect string 31 }{ 32 {"Using Request.Filename", ".testWithFilename", "/url-filename?filename=header-filename", ".testWithFilename"}, 33 {"Using Content-Disposition Header", "", "/url-filename?filename=.testWithHeaderFilename", ".testWithHeaderFilename"}, 34 {"Using Content-Disposition Header with target directory", ".test", "/url-filename?filename=header-filename", ".test/header-filename"}, 35 {"Using URL Path", "", "/.testWithURLFilename?params-filename", ".testWithURLFilename"}, 36 {"Using URL Path with target directory", ".test", "/url-filename?garbage", ".test/url-filename"}, 37 {"Failure", "", "", ""}, 38 } 39 40 err := os.Mkdir(".test", 0777) 41 if err != nil { 42 panic(err) 43 } 44 defer os.RemoveAll(".test") 45 46 for _, tc := range testCases { 47 t.Run(tc.Name, func(t *testing.T) { 48 req, _ := NewRequest(tc.Filename, ts.URL+tc.URL) 49 resp := DefaultClient.Do(req) 50 defer os.Remove(resp.Filename) 51 if err := resp.Err(); err != nil { 52 if tc.Expect != "" || err != ErrNoFilename { 53 panic(err) 54 } 55 } else { 56 if tc.Expect == "" { 57 t.Errorf("expected: %v, got: %v", ErrNoFilename, err) 58 } 59 } 60 61 if resp.Filename != tc.Expect { 62 t.Errorf("Filename mismatch. Expected '%s', got '%s'.", tc.Expect, resp.Filename) 63 } 64 65 testComplete(t, resp) 66 }) 67 } 68 } 69 70 // TestChecksums checks that checksum validation behaves as expected for valid 71 // and corrupted downloads. 72 func TestChecksums(t *testing.T) { 73 tests := []struct { 74 size int 75 hash hash.Hash 76 sum string 77 match bool 78 }{ 79 {128, md5.New(), "37eff01866ba3f538421b30b7cbefcac", true}, 80 {128, md5.New(), "37eff01866ba3f538421b30b7cbefcad", false}, 81 {1024, md5.New(), "b2ea9f7fcea831a4a63b213f41a8855b", true}, 82 {1024, md5.New(), "b2ea9f7fcea831a4a63b213f41a8855c", false}, 83 {1048576, md5.New(), "c35cc7d8d91728a0cb052831bc4ef372", true}, 84 {1048576, md5.New(), "c35cc7d8d91728a0cb052831bc4ef373", false}, 85 {128, sha1.New(), "e6434bc401f98603d7eda504790c98c67385d535", true}, 86 {128, sha1.New(), "e6434bc401f98603d7eda504790c98c67385d536", false}, 87 {1024, sha1.New(), "5b00669c480d5cffbdfa8bdba99561160f2d1b77", true}, 88 {1024, sha1.New(), "5b00669c480d5cffbdfa8bdba99561160f2d1b78", false}, 89 {1048576, sha1.New(), "ecfc8e86fdd83811f9cc9bf500993b63069923be", true}, 90 {1048576, sha1.New(), "ecfc8e86fdd83811f9cc9bf500993b63069923bf", false}, 91 {128, sha256.New(), "471fb943aa23c511f6f72f8d1652d9c880cfa392ad80503120547703e56a2be5", true}, 92 {128, sha256.New(), "471fb943aa23c511f6f72f8d1652d9c880cfa392ad80503120547703e56a2be4", false}, 93 {1024, sha256.New(), "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9", true}, 94 {1024, sha256.New(), "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c8", false}, 95 {1048576, sha256.New(), "fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c83", true}, 96 {1048576, sha256.New(), "fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c82", false}, 97 {128, sha512.New(), "1dffd5e3adb71d45d2245939665521ae001a317a03720a45732ba1900ca3b8351fc5c9b4ca513eba6f80bc7b1d1fdad4abd13491cb824d61b08d8c0e1561b3f7", true}, 98 {128, sha512.New(), "1dffd5e3adb71d45d2245939665521ae001a317a03720a45732ba1900ca3b8351fc5c9b4ca513eba6f80bc7b1d1fdad4abd13491cb824d61b08d8c0e1561b3f8", false}, 99 {1024, sha512.New(), "37f652be867f28ed033269cbba201af2112c2b3fd334a89fd2f757938ddee815787cc61d6e24a8a33340d0f7e86ffc058816b88530766ba6e231620a130b566c", true}, 100 {1024, sha512.New(), "37f652bf867f28ed033269cbba201af2112c2b3fd334a89fd2f757938ddee815787cc61d6e24a8a33340d0f7e86ffc058816b88530766ba6e231620a130b566d", false}, 101 {1048576, sha512.New(), "ac1d097b4ea6f6ad7ba640275b9ac290e4828cd760a0ebf76d555463a4f505f95df4f611629539a2dd1848e7c1304633baa1826462b3c87521c0c6e3469b67af", true}, 102 {1048576, sha512.New(), "ac1d097c4ea6f6ad7ba640275b9ac290e4828cd760a0ebf76d555463a4f505f95df4f611629539a2dd1848e7c1304633baa1826462b3c87521c0c6e3469b67af", false}, 103 } 104 105 for _, test := range tests { 106 var expect error 107 comparison := "Match" 108 if !test.match { 109 comparison = "Mismatch" 110 expect = ErrBadChecksum 111 } 112 113 t.Run(fmt.Sprintf("With%s%s", comparison, test.sum[:8]), func(t *testing.T) { 114 filename := fmt.Sprintf(".testChecksum-%s-%s", comparison, test.sum[:8]) 115 defer os.Remove(filename) 116 117 b, _ := hex.DecodeString(test.sum) 118 req, _ := NewRequest(filename, ts.URL+fmt.Sprintf("?size=%d", test.size)) 119 req.SetChecksum(test.hash, b, true) 120 121 resp := DefaultClient.Do(req) 122 err := resp.Err() 123 if err != expect { 124 t.Errorf("expected error: %v, got: %v", expect, err) 125 } 126 127 // ensure mismatch file was deleted 128 if !test.match { 129 if _, err := os.Stat(filename); err == nil { 130 t.Errorf("checksum failure not cleaned up: %s", filename) 131 } else if !os.IsNotExist(err) { 132 panic(err) 133 } 134 } 135 136 testComplete(t, resp) 137 }) 138 } 139 } 140 141 // TestContentLength ensures that ErrBadLength is returned if a server response 142 // does not match the requested length. 143 func TestContentLength(t *testing.T) { 144 size := int64(32768) 145 testCases := []struct { 146 Name string 147 URL string 148 Expect int64 149 Match bool 150 }{ 151 {"Good size in HEAD request", fmt.Sprintf("?size=%d", size), size, true}, 152 {"Good size in GET request", fmt.Sprintf("?nohead&size=%d", size), size, true}, 153 {"Bad size in HEAD request", fmt.Sprintf("?size=%d", size-1), size, false}, 154 {"Bad size in GET request", fmt.Sprintf("?nohead&size=%d", size-1), size, false}, 155 } 156 157 for _, tc := range testCases { 158 t.Run(tc.Name, func(t *testing.T) { 159 req, _ := NewRequest(".testSize-mismatch-head", ts.URL+tc.URL) 160 req.Size = size 161 162 resp := DefaultClient.Do(req) 163 defer os.Remove(resp.Filename) 164 err := resp.Err() 165 if tc.Match { 166 if err == ErrBadLength { 167 t.Errorf("error: %v", err) 168 } else if err != nil { 169 panic(err) 170 } else if resp.Size != size { 171 t.Errorf("expected %v bytes, got %v bytes", size, resp.Size) 172 } 173 } else { 174 if err == nil { 175 t.Errorf("expected: %v, got %v", ErrBadLength, err) 176 } else if err != ErrBadLength { 177 panic(err) 178 } 179 } 180 181 testComplete(t, resp) 182 }) 183 } 184 } 185 186 // TestAutoResume tests segmented downloading of a large file. 187 func TestAutoResume(t *testing.T) { 188 segs := 8 189 size := 1048576 190 sum, _ := hex.DecodeString("fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c83") 191 filename := ".testAutoResume" 192 193 defer os.Remove(filename) 194 195 for i := 0; i < segs; i++ { 196 segsize := (i + 1) * (size / segs) 197 t.Run(fmt.Sprintf("With%vBytes", segsize), func(t *testing.T) { 198 req, _ := NewRequest(filename, ts.URL+fmt.Sprintf("?size=%d", segsize)) 199 if i == segs-1 { 200 req.SetChecksum(sha256.New(), sum, false) 201 } 202 resp := DefaultClient.Do(req) 203 if err := resp.Err(); err != nil { 204 t.Errorf("error: %v", err) 205 return 206 } 207 if i > 0 && !resp.DidResume { 208 t.Errorf("expected Response.DidResume to be true") 209 } 210 testComplete(t, resp) 211 }) 212 } 213 214 t.Run("WithFailure", func(t *testing.T) { 215 // request smaller segment 216 req, _ := NewRequest(filename, ts.URL+fmt.Sprintf("?size=%d", size-1)) 217 resp := DefaultClient.Do(req) 218 if err := resp.Err(); err != ErrBadLength { 219 t.Errorf("expected ErrBadLength for smaller request, got: %v", err) 220 } 221 }) 222 223 t.Run("WithNoResume", func(t *testing.T) { 224 req, _ := NewRequest(filename, ts.URL+fmt.Sprintf("?size=%d", size+1)) 225 req.NoResume = true 226 resp := DefaultClient.Do(req) 227 if err := resp.Err(); err != nil { 228 panic(err) 229 } 230 if resp.DidResume == true { 231 t.Errorf("expected Response.DidResume to be false") 232 } 233 testComplete(t, resp) 234 }) 235 236 t.Run("WithNoResumeAndTruncate", func(t *testing.T) { 237 req, _ := NewRequest(filename, ts.URL+fmt.Sprintf("?size=%d", size-1)) 238 req.NoResume = true 239 resp := DefaultClient.Do(req) 240 if err := resp.Err(); err != nil { 241 t.Errorf("error in response: %v", err) 242 } 243 if resp.DidResume == true { 244 t.Errorf("expected Response.DidResume to be false") 245 } 246 testComplete(t, resp) 247 }) 248 // TODO: test when existing file is corrupted 249 } 250 251 func TestSkipExisting(t *testing.T) { 252 filename := ".testSkipExisting" 253 defer os.Remove(filename) 254 255 // download a file 256 req, _ := NewRequest(filename, ts.URL) 257 resp := DefaultClient.Do(req) 258 testComplete(t, resp) 259 260 // redownload 261 req, _ = NewRequest(filename, ts.URL) 262 resp = DefaultClient.Do(req) 263 testComplete(t, resp) 264 265 // ensure download was resumed 266 if !resp.DidResume { 267 t.Fatalf("Expected download to skip existing file, but it did not") 268 } 269 270 // ensure all bytes were resumed 271 if resp.Size == 0 || resp.Size != resp.bytesResumed { 272 t.Fatalf("Expected to skip %d bytes in redownload; got %d", resp.Size, resp.bytesResumed) 273 } 274 275 // ensure checksum is performed on pre-existing file 276 req, _ = NewRequest(filename, ts.URL) 277 req.SetChecksum(sha256.New(), []byte{0x01, 0x02, 0x03, 0x04}, true) 278 279 resp = DefaultClient.Do(req) 280 if err := resp.Err(); err != ErrBadChecksum { 281 t.Fatalf("Expected checksum error, got: %v", err) 282 } 283 } 284 285 // TestBatch executes multiple requests simultaneously and validates the 286 // responses. 287 func TestBatch(t *testing.T) { 288 tests := 32 289 size := 32768 290 sum := "e11360251d1173650cdcd20f111d8f1ca2e412f572e8b36a4dc067121c1799b8" 291 sumb, _ := hex.DecodeString(sum) 292 293 // test with 4 workers and with one per request 294 for _, workerCount := range []int{4, 0} { 295 // create requests 296 reqs := make([]*Request, tests) 297 for i := 0; i < len(reqs); i++ { 298 filename := fmt.Sprintf(".testBatch.%d", i+1) 299 reqs[i], _ = NewRequest(filename, ts.URL+fmt.Sprintf("/request_%d?size=%d&sleep=10", i, size)) 300 reqs[i].Label = fmt.Sprintf("Test %d", i+1) 301 reqs[i].SetChecksum(sha256.New(), sumb, false) 302 } 303 304 // batch run 305 responses := DefaultClient.DoBatch(workerCount, reqs...) 306 307 // listen for responses 308 Loop: 309 for i := 0; i < len(reqs); { 310 select { 311 case resp := <-responses: 312 if resp == nil { 313 break Loop 314 } 315 316 testComplete(t, resp) 317 if err := resp.Err(); err != nil { 318 t.Errorf("%s: %v", resp.Filename, err) 319 } 320 321 // remove test file 322 if resp.IsComplete() { 323 os.Remove(resp.Filename) // ignore errors 324 } 325 i++ 326 } 327 } 328 } 329 } 330 331 // TestCancelContext tests that a batch of requests can be cancel using a 332 // context.Context cancellation. Requests are cancelled in multiple states: 333 // in-progress and unstarted. 334 func TestCancelContext(t *testing.T) { 335 tests := 256 336 client := NewClient() 337 ctx, cancel := context.WithCancel(context.Background()) 338 defer cancel() 339 340 reqs := make([]*Request, tests) 341 for i := 0; i < tests; i++ { 342 req, _ := NewRequest("", fmt.Sprintf("%s/.testCancelContext%d?size=134217728", ts.URL, i)) 343 reqs[i] = req.WithContext(ctx) 344 } 345 346 respch := client.DoBatch(8, reqs...) 347 time.Sleep(time.Millisecond * 500) 348 cancel() 349 for resp := range respch { 350 defer os.Remove(resp.Filename) 351 352 // err should be context.Canceled or http.errRequestCanceled 353 if !strings.Contains(resp.Err().Error(), "canceled") { 354 t.Errorf("expected '%v', got '%v'", context.Canceled, resp.Err()) 355 } 356 } 357 } 358 359 // TestNestedDirectory tests that missing subdirectories are created. 360 func TestNestedDirectory(t *testing.T) { 361 dir := "./.testNested/one/two/three" 362 filename := ".testNestedFile" 363 expect := dir + "/" + filename 364 365 t.Run("Create", func(t *testing.T) { 366 req, _ := NewRequest(expect, ts.URL+"/"+filename) 367 resp := DefaultClient.Do(req) 368 if err := resp.Err(); err != nil { 369 panic(err) 370 } 371 defer os.RemoveAll("./.testNested/") 372 373 if resp.Filename != expect { 374 t.Errorf("expected nested Request.Filename to be %v, got %v", expect, resp.Filename) 375 } 376 }) 377 378 t.Run("No create", func(t *testing.T) { 379 req, _ := NewRequest(expect, ts.URL+"/"+filename) 380 req.NoCreateDirectories = true 381 382 resp := DefaultClient.Do(req) 383 err := resp.Err() 384 if !os.IsNotExist(err) { 385 t.Errorf("expected: %v, got: %v", os.ErrNotExist, err) 386 } 387 }) 388 } 389 390 // TestRemoteTime tests that the timestamp of the downloaded file can be set 391 // according to the timestamp of the remote file. 392 func TestRemoteTime(t *testing.T) { 393 filename := "./.testRemoteTime" 394 defer os.Remove(filename) 395 396 // random number between 0 and now 397 lastmod := rand.Int63n(time.Now().Unix()) 398 u := fmt.Sprintf("%s?lastmod=%d", ts.URL, lastmod) 399 req, _ := NewRequest(filename, u) 400 resp := DefaultClient.Do(req) 401 if err := resp.Err(); err != nil { 402 panic(err) 403 } 404 fi, err := os.Stat(resp.Filename) 405 if err != nil { 406 panic(err) 407 } 408 if fi.ModTime().Unix() != lastmod { 409 t.Errorf("expected %v, got %v", time.Unix(lastmod, 0), fi.ModTime()) 410 } 411 } 412 413 func TestResponseCode(t *testing.T) { 414 filename := "./.testResponseCode" 415 416 t.Run("With404", func(t *testing.T) { 417 defer os.Remove(filename) 418 req, _ := NewRequest(filename, ts.URL+"?status=404") 419 resp := DefaultClient.Do(req) 420 expect := StatusCodeError(http.StatusNotFound) 421 err := resp.Err() 422 if err != expect { 423 t.Errorf("expected %v, got '%v'", expect, err) 424 } 425 if !IsStatusCodeError(err) { 426 t.Errorf("expected IsStatusCodeError to return true for %T: %v", err, err) 427 } 428 }) 429 430 t.Run("WithIgnoreNon2XX", func(t *testing.T) { 431 defer os.Remove(filename) 432 req, _ := NewRequest(filename, ts.URL+"?status=404") 433 req.IgnoreBadStatusCodes = true 434 resp := DefaultClient.Do(req) 435 if err := resp.Err(); err != nil { 436 t.Errorf("expected nil, got '%v'", err) 437 } 438 }) 439 } 440 441 func TestBeforeCopyHook(t *testing.T) { 442 filename := "./.testBeforeCopy" 443 t.Run("Noop", func(t *testing.T) { 444 defer os.RemoveAll(filename) 445 called := false 446 req, _ := NewRequest(filename, ts.URL) 447 req.BeforeCopy = func(resp *Response) error { 448 called = true 449 if resp.IsComplete() { 450 t.Error("Response object passed to BeforeCopy hook has already been closed") 451 } 452 if resp.Progress() != 0 { 453 t.Error("Download progress already > 0 when BeforeCopy hook was called") 454 } 455 if resp.Duration() == 0 { 456 t.Error("Duration was zero when BeforeCopy was called") 457 } 458 if resp.BytesComplete() != 0 { 459 t.Error("BytesComplete already > 0 when BeforeCopy hook was called") 460 } 461 return nil 462 } 463 resp := DefaultClient.Do(req) 464 if err := resp.Err(); err != nil { 465 t.Errorf("unexpected error using BeforeCopy hook: %v", err) 466 } 467 testComplete(t, resp) 468 if !called { 469 t.Error("BeforeCopy hook was never called") 470 } 471 }) 472 473 t.Run("WithError", func(t *testing.T) { 474 defer os.RemoveAll(filename) 475 testError := errors.New("test") 476 req, _ := NewRequest(filename, ts.URL) 477 req.BeforeCopy = func(resp *Response) error { 478 return testError 479 } 480 resp := DefaultClient.Do(req) 481 if err := resp.Err(); err != testError { 482 t.Errorf("expected error '%v', got '%v'", testError, err) 483 } 484 if resp.BytesComplete() != 0 { 485 t.Errorf("expected 0 bytes completed for canceled BeforeCopy hook, got %d", 486 resp.BytesComplete()) 487 } 488 testComplete(t, resp) 489 }) 490 } 491 492 func TestAfterCopyHook(t *testing.T) { 493 filename := "./.testAfterCopy" 494 t.Run("Noop", func(t *testing.T) { 495 defer os.RemoveAll(filename) 496 called := false 497 req, _ := NewRequest(filename, ts.URL) 498 req.AfterCopy = func(resp *Response) error { 499 called = true 500 if resp.IsComplete() { 501 t.Error("Response object passed to AfterCopy hook has already been closed") 502 } 503 if resp.Progress() <= 0 { 504 t.Error("Download progress was 0 when AfterCopy hook was called") 505 } 506 if resp.Duration() == 0 { 507 t.Error("Duration was zero when AfterCopy was called") 508 } 509 if resp.BytesComplete() <= 0 { 510 t.Error("BytesComplete was 0 when AfterCopy hook was called") 511 } 512 return nil 513 } 514 resp := DefaultClient.Do(req) 515 if err := resp.Err(); err != nil { 516 t.Errorf("unexpected error using AfterCopy hook: %v", err) 517 } 518 testComplete(t, resp) 519 if !called { 520 t.Error("AfterCopy hook was never called") 521 } 522 }) 523 524 t.Run("WithError", func(t *testing.T) { 525 defer os.RemoveAll(filename) 526 testError := errors.New("test") 527 req, _ := NewRequest(filename, ts.URL) 528 req.AfterCopy = func(resp *Response) error { 529 return testError 530 } 531 resp := DefaultClient.Do(req) 532 if err := resp.Err(); err != testError { 533 t.Errorf("expected error '%v', got '%v'", testError, err) 534 } 535 if resp.BytesComplete() <= 0 { 536 t.Errorf("ByteCompleted was %d after AfterCopy hook was called", 537 resp.BytesComplete()) 538 } 539 testComplete(t, resp) 540 }) 541 } 542 543 func TestIssue37(t *testing.T) { 544 // ref: https://github.com/cavaliercoder/grab/issues/37 545 filename := "./.testIssue37" 546 largeSize := 2097152 547 smallSize := 1048576 548 defer os.RemoveAll(filename) 549 550 // download large file 551 req, _ := NewRequest(filename, fmt.Sprintf("%s?size=%d", ts.URL, largeSize)) 552 resp := DefaultClient.Do(req) 553 if err := resp.Err(); err != nil { 554 t.Fatal(err) 555 } 556 557 // download new, smaller version of same file 558 req, _ = NewRequest(filename, fmt.Sprintf("%s?size=%d", ts.URL, smallSize)) 559 req.NoResume = true 560 resp = DefaultClient.Do(req) 561 if err := resp.Err(); err != nil { 562 t.Fatal(err) 563 } 564 565 // local file should have truncated and not resumed 566 if resp.DidResume { 567 t.Errorf("expected download to truncate, resumed instead") 568 } 569 570 fi, err := os.Stat(filename) 571 if err != nil { 572 t.Fatal(err) 573 } 574 575 if fi.Size() != int64(smallSize) { 576 t.Errorf("expected file size %d, got %d", smallSize, fi.Size()) 577 } 578 } 579 580 // TestHeadBadStatus validates that HEAD requests that return non-200 can be 581 // ignored and succeed if the GET requests succeeeds. 582 // 583 // Fixes: https://github.com/cavaliercoder/grab/issues/43 584 func TestHeadBadStatus(t *testing.T) { 585 expect := http.StatusOK 586 filename := ".testIssue43" 587 testURL := fmt.Sprintf( 588 "%s/%s?headStatus=%d", 589 ts.URL, 590 filename, 591 http.StatusForbidden) 592 req, _ := NewRequest("", testURL) 593 resp := DefaultClient.Do(req) 594 if err := resp.Err(); err != nil { 595 t.Fatal(err) 596 } 597 if resp.HTTPResponse.StatusCode != expect { 598 t.Errorf( 599 "expected status code: %d, got:% d", 600 expect, 601 resp.HTTPResponse.StatusCode) 602 } 603 }