github.com/aavshr/aws-sdk-go@v1.41.3/service/s3/s3manager/batch_test.go (about) 1 package s3manager 2 3 import ( 4 "bytes" 5 "errors" 6 "io/ioutil" 7 "net/http" 8 "net/http/httptest" 9 "strings" 10 "testing" 11 12 "github.com/aavshr/aws-sdk-go/aws" 13 "github.com/aavshr/aws-sdk-go/aws/awserr" 14 "github.com/aavshr/aws-sdk-go/aws/credentials" 15 "github.com/aavshr/aws-sdk-go/aws/request" 16 "github.com/aavshr/aws-sdk-go/awstesting/unit" 17 "github.com/aavshr/aws-sdk-go/service/s3" 18 "github.com/aavshr/aws-sdk-go/service/s3/s3iface" 19 ) 20 21 func TestHasParity(t *testing.T) { 22 cases := []struct { 23 o1 *s3.DeleteObjectsInput 24 o2 BatchDeleteObject 25 expected bool 26 }{ 27 { 28 &s3.DeleteObjectsInput{}, 29 BatchDeleteObject{ 30 Object: &s3.DeleteObjectInput{}, 31 }, 32 true, 33 }, 34 { 35 &s3.DeleteObjectsInput{ 36 Bucket: aws.String("foo"), 37 }, 38 BatchDeleteObject{ 39 Object: &s3.DeleteObjectInput{ 40 Bucket: aws.String("bar"), 41 }, 42 }, 43 false, 44 }, 45 { 46 &s3.DeleteObjectsInput{}, 47 BatchDeleteObject{ 48 Object: &s3.DeleteObjectInput{ 49 Bucket: aws.String("foo"), 50 }, 51 }, 52 false, 53 }, 54 { 55 &s3.DeleteObjectsInput{ 56 Bucket: aws.String("foo"), 57 }, 58 BatchDeleteObject{ 59 Object: &s3.DeleteObjectInput{}, 60 }, 61 false, 62 }, 63 { 64 &s3.DeleteObjectsInput{ 65 MFA: aws.String("foo"), 66 }, 67 BatchDeleteObject{ 68 Object: &s3.DeleteObjectInput{ 69 MFA: aws.String("bar"), 70 }, 71 }, 72 false, 73 }, 74 { 75 &s3.DeleteObjectsInput{}, 76 BatchDeleteObject{ 77 Object: &s3.DeleteObjectInput{ 78 MFA: aws.String("foo"), 79 }, 80 }, 81 false, 82 }, 83 { 84 &s3.DeleteObjectsInput{ 85 MFA: aws.String("foo"), 86 }, 87 BatchDeleteObject{ 88 Object: &s3.DeleteObjectInput{}, 89 }, 90 false, 91 }, 92 { 93 &s3.DeleteObjectsInput{ 94 RequestPayer: aws.String("foo"), 95 }, 96 BatchDeleteObject{ 97 Object: &s3.DeleteObjectInput{ 98 RequestPayer: aws.String("bar"), 99 }, 100 }, 101 false, 102 }, 103 { 104 &s3.DeleteObjectsInput{}, 105 BatchDeleteObject{ 106 Object: &s3.DeleteObjectInput{ 107 RequestPayer: aws.String("foo"), 108 }, 109 }, 110 false, 111 }, 112 { 113 &s3.DeleteObjectsInput{ 114 RequestPayer: aws.String("foo"), 115 }, 116 BatchDeleteObject{ 117 Object: &s3.DeleteObjectInput{}, 118 }, 119 false, 120 }, 121 } 122 123 for i, c := range cases { 124 if result := hasParity(c.o1, c.o2); result != c.expected { 125 t.Errorf("Case %d: expected %t, but received %t\n", i, c.expected, result) 126 } 127 } 128 } 129 130 func TestBatchDelete(t *testing.T) { 131 cases := []struct { 132 objects []BatchDeleteObject 133 size int 134 expected int 135 }{ 136 { 137 []BatchDeleteObject{ 138 { 139 Object: &s3.DeleteObjectInput{ 140 Key: aws.String("1"), 141 Bucket: aws.String("bucket1"), 142 }, 143 }, 144 { 145 Object: &s3.DeleteObjectInput{ 146 Key: aws.String("2"), 147 Bucket: aws.String("bucket2"), 148 }, 149 }, 150 { 151 Object: &s3.DeleteObjectInput{ 152 Key: aws.String("3"), 153 Bucket: aws.String("bucket3"), 154 }, 155 }, 156 { 157 Object: &s3.DeleteObjectInput{ 158 Key: aws.String("4"), 159 Bucket: aws.String("bucket4"), 160 }, 161 }, 162 }, 163 1, 164 4, 165 }, 166 { 167 []BatchDeleteObject{ 168 { 169 Object: &s3.DeleteObjectInput{ 170 Key: aws.String("1"), 171 Bucket: aws.String("bucket1"), 172 }, 173 }, 174 { 175 Object: &s3.DeleteObjectInput{ 176 Key: aws.String("2"), 177 Bucket: aws.String("bucket1"), 178 }, 179 }, 180 { 181 Object: &s3.DeleteObjectInput{ 182 Key: aws.String("3"), 183 Bucket: aws.String("bucket3"), 184 }, 185 }, 186 { 187 Object: &s3.DeleteObjectInput{ 188 Key: aws.String("4"), 189 Bucket: aws.String("bucket3"), 190 }, 191 }, 192 }, 193 1, 194 4, 195 }, 196 { 197 []BatchDeleteObject{ 198 { 199 Object: &s3.DeleteObjectInput{ 200 Key: aws.String("1"), 201 Bucket: aws.String("bucket1"), 202 }, 203 }, 204 { 205 Object: &s3.DeleteObjectInput{ 206 Key: aws.String("2"), 207 Bucket: aws.String("bucket1"), 208 }, 209 }, 210 { 211 Object: &s3.DeleteObjectInput{ 212 Key: aws.String("3"), 213 Bucket: aws.String("bucket3"), 214 }, 215 }, 216 { 217 Object: &s3.DeleteObjectInput{ 218 Key: aws.String("4"), 219 Bucket: aws.String("bucket3"), 220 }, 221 }, 222 }, 223 4, 224 2, 225 }, 226 { 227 []BatchDeleteObject{ 228 { 229 Object: &s3.DeleteObjectInput{ 230 Key: aws.String("1"), 231 Bucket: aws.String("bucket1"), 232 }, 233 }, 234 { 235 Object: &s3.DeleteObjectInput{ 236 Key: aws.String("2"), 237 Bucket: aws.String("bucket1"), 238 }, 239 }, 240 { 241 Object: &s3.DeleteObjectInput{ 242 Key: aws.String("3"), 243 Bucket: aws.String("bucket3"), 244 }, 245 }, 246 { 247 Object: &s3.DeleteObjectInput{ 248 Key: aws.String("4"), 249 Bucket: aws.String("bucket3"), 250 }, 251 }, 252 }, 253 10, 254 2, 255 }, 256 { 257 []BatchDeleteObject{ 258 { 259 Object: &s3.DeleteObjectInput{ 260 Key: aws.String("1"), 261 Bucket: aws.String("bucket1"), 262 }, 263 }, 264 { 265 Object: &s3.DeleteObjectInput{ 266 Key: aws.String("2"), 267 Bucket: aws.String("bucket1"), 268 }, 269 }, 270 { 271 Object: &s3.DeleteObjectInput{ 272 Key: aws.String("3"), 273 Bucket: aws.String("bucket1"), 274 }, 275 }, 276 { 277 Object: &s3.DeleteObjectInput{ 278 Key: aws.String("4"), 279 Bucket: aws.String("bucket3"), 280 }, 281 }, 282 }, 283 2, 284 3, 285 }, 286 } 287 288 count := 0 289 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 290 w.WriteHeader(http.StatusNoContent) 291 count++ 292 })) 293 defer server.Close() 294 295 svc := &mockS3Client{S3: buildS3SvcClient(server.URL)} 296 for i, c := range cases { 297 batcher := BatchDelete{ 298 Client: svc, 299 BatchSize: c.size, 300 } 301 302 if err := batcher.Delete(aws.BackgroundContext(), &DeleteObjectsIterator{Objects: c.objects}); err != nil { 303 panic(err) 304 } 305 306 if count != c.expected { 307 t.Errorf("Case %d: expected %d, but received %d", i, c.expected, count) 308 } 309 310 count = 0 311 } 312 } 313 314 func TestBatchDeleteError(t *testing.T) { 315 cases := []struct { 316 objects []BatchDeleteObject 317 output s3.DeleteObjectsOutput 318 size int 319 expectedErrCode string 320 expectedErrMessage string 321 }{ 322 { 323 []BatchDeleteObject{ 324 { 325 Object: &s3.DeleteObjectInput{ 326 Key: aws.String("1"), 327 Bucket: aws.String("bucket1"), 328 }, 329 }, 330 }, 331 s3.DeleteObjectsOutput{ 332 Errors: []*s3.Error{ 333 { 334 Code: aws.String("foo code"), 335 Message: aws.String("foo error"), 336 }, 337 }, 338 }, 339 1, 340 "foo code", 341 "foo error", 342 }, 343 { 344 []BatchDeleteObject{ 345 { 346 Object: &s3.DeleteObjectInput{ 347 Key: aws.String("1"), 348 Bucket: aws.String("bucket1"), 349 }, 350 }, 351 }, 352 s3.DeleteObjectsOutput{ 353 Errors: []*s3.Error{ 354 {}, 355 }, 356 }, 357 1, 358 ErrDeleteBatchFailCode, 359 errDefaultDeleteBatchMessage, 360 }, 361 } 362 363 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 364 w.WriteHeader(http.StatusNoContent) 365 })) 366 defer server.Close() 367 368 index := 0 369 svc := &mockS3Client{ 370 S3: buildS3SvcClient(server.URL), 371 deleteObjects: func(input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) { 372 output := &cases[index].output 373 index++ 374 return output, nil 375 }, 376 } 377 for _, c := range cases { 378 batcher := BatchDelete{ 379 Client: svc, 380 BatchSize: c.size, 381 } 382 383 err := batcher.Delete(aws.BackgroundContext(), &DeleteObjectsIterator{Objects: c.objects}) 384 if err == nil { 385 t.Errorf("expected error, but received none") 386 } 387 388 berr := err.(*BatchError) 389 390 if len(berr.Errors) != 1 { 391 t.Errorf("expected 1 error, but received %d", len(berr.Errors)) 392 } 393 394 aerr := berr.Errors[0].OrigErr.(awserr.Error) 395 if e, a := c.expectedErrCode, aerr.Code(); e != a { 396 t.Errorf("expected %q, but received %q", e, a) 397 } 398 399 if e, a := c.expectedErrMessage, aerr.Message(); e != a { 400 t.Errorf("expected %q, but received %q", e, a) 401 } 402 } 403 } 404 405 type mockS3Client struct { 406 *s3.S3 407 index int 408 objects []*s3.ListObjectsOutput 409 deleteObjects func(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) 410 } 411 412 func (client *mockS3Client) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { 413 object := client.objects[client.index] 414 client.index++ 415 return object, nil 416 } 417 418 func (client *mockS3Client) DeleteObjects(input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) { 419 if client.deleteObjects == nil { 420 return client.S3.DeleteObjectsWithContext(aws.BackgroundContext(), input) 421 } 422 423 return client.deleteObjects(input) 424 } 425 426 func (client *mockS3Client) DeleteObjectsWithContext(ctx aws.Context, input *s3.DeleteObjectsInput, opt ...request.Option) (*s3.DeleteObjectsOutput, error) { 427 if client.deleteObjects == nil { 428 return client.S3.DeleteObjectsWithContext(ctx, input) 429 } 430 431 return client.deleteObjects(input) 432 } 433 434 func TestNilOrigError(t *testing.T) { 435 err := Error{ 436 Bucket: aws.String("bucket"), 437 Key: aws.String("key"), 438 } 439 errStr := err.Error() 440 const expected1 = `failed to perform batch operation on "key" to "bucket"` 441 if errStr != expected1 { 442 t.Errorf("Expected %s, but received %s", expected1, errStr) 443 } 444 445 err = Error{ 446 OrigErr: errors.New("foo"), 447 Bucket: aws.String("bucket"), 448 Key: aws.String("key"), 449 } 450 errStr = err.Error() 451 const expected2 = "failed to perform batch operation on \"key\" to \"bucket\":\nfoo" 452 if errStr != expected2 { 453 t.Errorf("Expected %s, but received %s", expected2, errStr) 454 } 455 456 } 457 458 func TestBatchDeleteList(t *testing.T) { 459 count := 0 460 461 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 462 w.WriteHeader(http.StatusNoContent) 463 count++ 464 })) 465 defer server.Close() 466 467 objects := []*s3.ListObjectsOutput{ 468 { 469 Contents: []*s3.Object{ 470 { 471 Key: aws.String("1"), 472 }, 473 }, 474 NextMarker: aws.String("marker"), 475 IsTruncated: aws.Bool(true), 476 }, 477 { 478 Contents: []*s3.Object{ 479 { 480 Key: aws.String("2"), 481 }, 482 }, 483 NextMarker: aws.String("marker"), 484 IsTruncated: aws.Bool(true), 485 }, 486 { 487 Contents: []*s3.Object{ 488 { 489 Key: aws.String("3"), 490 }, 491 }, 492 IsTruncated: aws.Bool(false), 493 }, 494 } 495 496 svc := &mockS3Client{S3: buildS3SvcClient(server.URL), objects: objects} 497 batcher := BatchDelete{ 498 Client: svc, 499 BatchSize: 1, 500 } 501 502 input := &s3.ListObjectsInput{ 503 Bucket: aws.String("bucket"), 504 } 505 iter := &DeleteListIterator{ 506 Bucket: input.Bucket, 507 Paginator: request.Pagination{ 508 NewRequest: func() (*request.Request, error) { 509 var inCpy *s3.ListObjectsInput 510 if input != nil { 511 tmp := *input 512 inCpy = &tmp 513 } 514 req, _ := svc.ListObjectsRequest(inCpy) 515 req.Handlers.Clear() 516 output, _ := svc.ListObjects(inCpy) 517 req.Data = output 518 return req, nil 519 }, 520 }, 521 } 522 523 if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil { 524 t.Error(err) 525 } 526 527 if count != len(objects) { 528 t.Errorf("Expected %d, but received %d", len(objects), count) 529 } 530 } 531 532 func buildS3SvcClient(u string) *s3.S3 { 533 return s3.New(unit.Session, &aws.Config{ 534 Endpoint: aws.String(u), 535 S3ForcePathStyle: aws.Bool(true), 536 DisableSSL: aws.Bool(true), 537 Credentials: credentials.NewStaticCredentials("AKID", "SECRET", "SESSION"), 538 }) 539 540 } 541 542 func TestBatchDeleteList_EmptyListObjects(t *testing.T) { 543 count := 0 544 545 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 546 w.WriteHeader(http.StatusNoContent) 547 count++ 548 })) 549 defer server.Close() 550 551 svc := &mockS3Client{S3: buildS3SvcClient(server.URL)} 552 batcher := BatchDelete{ 553 Client: svc, 554 } 555 556 input := &s3.ListObjectsInput{ 557 Bucket: aws.String("bucket"), 558 } 559 560 // Test DeleteListIterator in the case when the ListObjectsRequest responds 561 // with an empty listing. 562 563 // We need a new iterator with a fresh Pagination since 564 // Pagination.HasNextPage() is always true the first time Pagination.Next() 565 // called on it 566 iter := &DeleteListIterator{ 567 Bucket: input.Bucket, 568 Paginator: request.Pagination{ 569 NewRequest: func() (*request.Request, error) { 570 req, _ := svc.ListObjectsRequest(input) 571 // Simulate empty listing 572 req.Data = &s3.ListObjectsOutput{Contents: []*s3.Object{}} 573 return req, nil 574 }, 575 }, 576 } 577 578 if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil { 579 t.Error(err) 580 } 581 if count != 1 { 582 t.Errorf("expect count to be 1, got %d", count) 583 } 584 } 585 586 func TestBatchDownload(t *testing.T) { 587 count := 0 588 expected := []struct { 589 bucket, key string 590 }{ 591 { 592 key: "1", 593 bucket: "bucket1", 594 }, 595 { 596 key: "2", 597 bucket: "bucket2", 598 }, 599 { 600 key: "3", 601 bucket: "bucket3", 602 }, 603 { 604 key: "4", 605 bucket: "bucket4", 606 }, 607 } 608 609 received := []struct { 610 bucket, key string 611 }{} 612 613 payload := []string{ 614 "1", 615 "2", 616 "3", 617 "4", 618 } 619 620 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 621 urlParts := strings.Split(r.URL.String(), "/") 622 received = append(received, struct{ bucket, key string }{urlParts[1], urlParts[2]}) 623 w.Write([]byte(payload[count])) 624 count++ 625 })) 626 defer server.Close() 627 628 svc := NewDownloaderWithClient(buildS3SvcClient(server.URL)) 629 630 objects := []BatchDownloadObject{ 631 { 632 Object: &s3.GetObjectInput{ 633 Key: aws.String("1"), 634 Bucket: aws.String("bucket1"), 635 }, 636 Writer: aws.NewWriteAtBuffer(make([]byte, 128)), 637 }, 638 { 639 Object: &s3.GetObjectInput{ 640 Key: aws.String("2"), 641 Bucket: aws.String("bucket2"), 642 }, 643 Writer: aws.NewWriteAtBuffer(make([]byte, 128)), 644 }, 645 { 646 Object: &s3.GetObjectInput{ 647 Key: aws.String("3"), 648 Bucket: aws.String("bucket3"), 649 }, 650 Writer: aws.NewWriteAtBuffer(make([]byte, 128)), 651 }, 652 { 653 Object: &s3.GetObjectInput{ 654 Key: aws.String("4"), 655 Bucket: aws.String("bucket4"), 656 }, 657 Writer: aws.NewWriteAtBuffer(make([]byte, 128)), 658 }, 659 } 660 661 iter := &DownloadObjectsIterator{Objects: objects} 662 if err := svc.DownloadWithIterator(aws.BackgroundContext(), iter); err != nil { 663 panic(err) 664 } 665 666 if count != len(objects) { 667 t.Errorf("Expected %d, but received %d", len(objects), count) 668 } 669 670 if len(expected) != len(received) { 671 t.Errorf("Expected %d, but received %d", len(expected), len(received)) 672 } 673 674 for i := 0; i < len(expected); i++ { 675 if expected[i].key != received[i].key { 676 t.Errorf("Expected %q, but received %q", expected[i].key, received[i].key) 677 } 678 679 if expected[i].bucket != received[i].bucket { 680 t.Errorf("Expected %q, but received %q", expected[i].bucket, received[i].bucket) 681 } 682 } 683 684 for i, p := range payload { 685 b := iter.Objects[i].Writer.(*aws.WriteAtBuffer).Bytes() 686 b = bytes.Trim(b, "\x00") 687 688 if string(b) != p { 689 t.Errorf("Expected %q, but received %q", p, b) 690 } 691 } 692 } 693 694 func TestBatchUpload(t *testing.T) { 695 count := 0 696 expected := []struct { 697 bucket, key string 698 reqBody string 699 }{ 700 { 701 key: "1", 702 bucket: "bucket1", 703 reqBody: "1", 704 }, 705 { 706 key: "2", 707 bucket: "bucket2", 708 reqBody: "2", 709 }, 710 { 711 key: "3", 712 bucket: "bucket3", 713 reqBody: "3", 714 }, 715 { 716 key: "4", 717 bucket: "bucket4", 718 reqBody: "4", 719 }, 720 } 721 722 received := []struct { 723 bucket, key, reqBody string 724 }{} 725 726 payload := []string{ 727 "a", 728 "b", 729 "c", 730 "d", 731 } 732 733 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 734 urlParts := strings.Split(r.URL.String(), "/") 735 736 b, err := ioutil.ReadAll(r.Body) 737 if err != nil { 738 t.Error(err) 739 } 740 741 received = append(received, struct{ bucket, key, reqBody string }{urlParts[1], urlParts[2], string(b)}) 742 w.Write([]byte(payload[count])) 743 744 count++ 745 })) 746 defer server.Close() 747 748 svc := NewUploaderWithClient(buildS3SvcClient(server.URL)) 749 750 objects := []BatchUploadObject{ 751 { 752 Object: &UploadInput{ 753 Key: aws.String("1"), 754 Bucket: aws.String("bucket1"), 755 Body: bytes.NewBuffer([]byte("1")), 756 }, 757 }, 758 { 759 Object: &UploadInput{ 760 Key: aws.String("2"), 761 Bucket: aws.String("bucket2"), 762 Body: bytes.NewBuffer([]byte("2")), 763 }, 764 }, 765 { 766 Object: &UploadInput{ 767 Key: aws.String("3"), 768 Bucket: aws.String("bucket3"), 769 Body: bytes.NewBuffer([]byte("3")), 770 }, 771 }, 772 { 773 Object: &UploadInput{ 774 Key: aws.String("4"), 775 Bucket: aws.String("bucket4"), 776 Body: bytes.NewBuffer([]byte("4")), 777 }, 778 }, 779 } 780 781 iter := &UploadObjectsIterator{Objects: objects} 782 if err := svc.UploadWithIterator(aws.BackgroundContext(), iter); err != nil { 783 panic(err) 784 } 785 786 if count != len(objects) { 787 t.Errorf("Expected %d, but received %d", len(objects), count) 788 } 789 790 if len(expected) != len(received) { 791 t.Errorf("Expected %d, but received %d", len(expected), len(received)) 792 } 793 794 for i := 0; i < len(expected); i++ { 795 if expected[i].key != received[i].key { 796 t.Errorf("Expected %q, but received %q", expected[i].key, received[i].key) 797 } 798 799 if expected[i].bucket != received[i].bucket { 800 t.Errorf("Expected %q, but received %q", expected[i].bucket, received[i].bucket) 801 } 802 803 if expected[i].reqBody != received[i].reqBody { 804 t.Errorf("Expected %q, but received %q", expected[i].reqBody, received[i].reqBody) 805 } 806 } 807 } 808 809 type mockClient struct { 810 s3iface.S3API 811 Put func() (*s3.PutObjectOutput, error) 812 Get func() (*s3.GetObjectOutput, error) 813 List func() (*s3.ListObjectsOutput, error) 814 responses []response 815 } 816 817 type response struct { 818 out interface{} 819 err error 820 } 821 822 func (client *mockClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { 823 return client.Put() 824 } 825 826 func (client *mockClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { 827 req, _ := client.S3API.PutObjectRequest(input) 828 req.Handlers.Clear() 829 req.Data, req.Error = client.Put() 830 return req, req.Data.(*s3.PutObjectOutput) 831 } 832 833 func (client *mockClient) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { 834 return client.List() 835 } 836 837 func (client *mockClient) ListObjectsRequest(input *s3.ListObjectsInput) (*request.Request, *s3.ListObjectsOutput) { 838 req, _ := client.S3API.ListObjectsRequest(input) 839 req.Handlers.Clear() 840 req.Data, req.Error = client.List() 841 return req, req.Data.(*s3.ListObjectsOutput) 842 } 843 844 func TestBatchError(t *testing.T) { 845 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 846 })) 847 defer server.Close() 848 849 index := 0 850 responses := []response{ 851 { 852 &s3.PutObjectOutput{}, 853 errors.New("Foo"), 854 }, 855 { 856 &s3.PutObjectOutput{}, 857 nil, 858 }, 859 { 860 &s3.PutObjectOutput{}, 861 nil, 862 }, 863 { 864 &s3.PutObjectOutput{}, 865 errors.New("Bar"), 866 }, 867 } 868 869 svc := &mockClient{ 870 S3API: buildS3SvcClient(server.URL), 871 Put: func() (*s3.PutObjectOutput, error) { 872 resp := responses[index] 873 index++ 874 return resp.out.(*s3.PutObjectOutput), resp.err 875 }, 876 List: func() (*s3.ListObjectsOutput, error) { 877 resp := responses[index] 878 index++ 879 return resp.out.(*s3.ListObjectsOutput), resp.err 880 }, 881 } 882 uploader := NewUploaderWithClient(svc) 883 884 objects := []BatchUploadObject{ 885 { 886 Object: &UploadInput{ 887 Key: aws.String("1"), 888 Bucket: aws.String("bucket1"), 889 Body: bytes.NewBuffer([]byte("1")), 890 }, 891 }, 892 { 893 Object: &UploadInput{ 894 Key: aws.String("2"), 895 Bucket: aws.String("bucket2"), 896 Body: bytes.NewBuffer([]byte("2")), 897 }, 898 }, 899 { 900 Object: &UploadInput{ 901 Key: aws.String("3"), 902 Bucket: aws.String("bucket3"), 903 Body: bytes.NewBuffer([]byte("3")), 904 }, 905 }, 906 { 907 Object: &UploadInput{ 908 Key: aws.String("4"), 909 Bucket: aws.String("bucket4"), 910 Body: bytes.NewBuffer([]byte("4")), 911 }, 912 }, 913 } 914 915 iter := &UploadObjectsIterator{Objects: objects} 916 if err := uploader.UploadWithIterator(aws.BackgroundContext(), iter); err != nil { 917 if bErr, ok := err.(*BatchError); !ok { 918 t.Error("Expected BatchError, but received other") 919 } else { 920 if len(bErr.Errors) != 2 { 921 t.Errorf("Expected 2 errors, but received %d", len(bErr.Errors)) 922 } 923 924 expected := []struct { 925 bucket, key string 926 }{ 927 { 928 "bucket1", 929 "1", 930 }, 931 { 932 "bucket4", 933 "4", 934 }, 935 } 936 for i, expect := range expected { 937 if *bErr.Errors[i].Bucket != expect.bucket { 938 t.Errorf("Case %d: Invalid bucket expected %s, but received %s", i, expect.bucket, *bErr.Errors[i].Bucket) 939 } 940 941 if *bErr.Errors[i].Key != expect.key { 942 t.Errorf("Case %d: Invalid key expected %s, but received %s", i, expect.key, *bErr.Errors[i].Key) 943 } 944 } 945 } 946 } else { 947 t.Error("Expected error, but received nil") 948 } 949 950 if index != len(objects) { 951 t.Errorf("Expected %d, but received %d", len(objects), index) 952 } 953 954 } 955 956 type testAfterDeleteIter struct { 957 afterDelete bool 958 afterDownload bool 959 afterUpload bool 960 next bool 961 } 962 963 func (iter *testAfterDeleteIter) Next() bool { 964 next := !iter.next 965 iter.next = !iter.next 966 return next 967 } 968 969 func (iter *testAfterDeleteIter) Err() error { 970 return nil 971 } 972 973 func (iter *testAfterDeleteIter) DeleteObject() BatchDeleteObject { 974 return BatchDeleteObject{ 975 Object: &s3.DeleteObjectInput{ 976 Bucket: aws.String("foo"), 977 Key: aws.String("foo"), 978 }, 979 After: func() error { 980 iter.afterDelete = true 981 return nil 982 }, 983 } 984 } 985 986 type testAfterDownloadIter struct { 987 afterDownload bool 988 afterUpload bool 989 next bool 990 } 991 992 func (iter *testAfterDownloadIter) Next() bool { 993 next := !iter.next 994 iter.next = !iter.next 995 return next 996 } 997 998 func (iter *testAfterDownloadIter) Err() error { 999 return nil 1000 } 1001 1002 func (iter *testAfterDownloadIter) DownloadObject() BatchDownloadObject { 1003 return BatchDownloadObject{ 1004 Object: &s3.GetObjectInput{ 1005 Bucket: aws.String("foo"), 1006 Key: aws.String("foo"), 1007 }, 1008 Writer: aws.NewWriteAtBuffer([]byte{}), 1009 After: func() error { 1010 iter.afterDownload = true 1011 return nil 1012 }, 1013 } 1014 } 1015 1016 type testAfterUploadIter struct { 1017 afterUpload bool 1018 next bool 1019 } 1020 1021 func (iter *testAfterUploadIter) Next() bool { 1022 next := !iter.next 1023 iter.next = !iter.next 1024 return next 1025 } 1026 1027 func (iter *testAfterUploadIter) Err() error { 1028 return nil 1029 } 1030 1031 func (iter *testAfterUploadIter) UploadObject() BatchUploadObject { 1032 return BatchUploadObject{ 1033 Object: &UploadInput{ 1034 Bucket: aws.String("foo"), 1035 Key: aws.String("foo"), 1036 Body: strings.NewReader("bar"), 1037 }, 1038 After: func() error { 1039 iter.afterUpload = true 1040 return nil 1041 }, 1042 } 1043 } 1044 1045 func TestAfter(t *testing.T) { 1046 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 1047 })) 1048 defer server.Close() 1049 1050 index := 0 1051 responses := []response{ 1052 { 1053 &s3.PutObjectOutput{}, 1054 nil, 1055 }, 1056 { 1057 &s3.GetObjectOutput{}, 1058 nil, 1059 }, 1060 { 1061 &s3.DeleteObjectOutput{}, 1062 nil, 1063 }, 1064 } 1065 1066 svc := &mockClient{ 1067 S3API: buildS3SvcClient(server.URL), 1068 Put: func() (*s3.PutObjectOutput, error) { 1069 resp := responses[index] 1070 index++ 1071 return resp.out.(*s3.PutObjectOutput), resp.err 1072 }, 1073 Get: func() (*s3.GetObjectOutput, error) { 1074 resp := responses[index] 1075 index++ 1076 return resp.out.(*s3.GetObjectOutput), resp.err 1077 }, 1078 List: func() (*s3.ListObjectsOutput, error) { 1079 resp := responses[index] 1080 index++ 1081 return resp.out.(*s3.ListObjectsOutput), resp.err 1082 }, 1083 } 1084 uploader := NewUploaderWithClient(svc) 1085 downloader := NewDownloaderWithClient(svc) 1086 deleter := NewBatchDeleteWithClient(svc) 1087 1088 deleteIter := &testAfterDeleteIter{} 1089 downloadIter := &testAfterDownloadIter{} 1090 uploadIter := &testAfterUploadIter{} 1091 1092 if err := uploader.UploadWithIterator(aws.BackgroundContext(), uploadIter); err != nil { 1093 t.Error(err) 1094 } 1095 1096 if err := downloader.DownloadWithIterator(aws.BackgroundContext(), downloadIter); err != nil { 1097 t.Error(err) 1098 } 1099 1100 if err := deleter.Delete(aws.BackgroundContext(), deleteIter); err != nil { 1101 t.Error(err) 1102 } 1103 1104 if !deleteIter.afterDelete { 1105 t.Error("Expected 'afterDelete' to be true, but received false") 1106 } 1107 1108 if !downloadIter.afterDownload { 1109 t.Error("Expected 'afterDownload' to be true, but received false") 1110 } 1111 1112 if !uploadIter.afterUpload { 1113 t.Error("Expected 'afterUpload' to be true, but received false") 1114 } 1115 }