github.com/aavshr/aws-sdk-go@v1.41.3/service/s3/s3manager/batch_test.go (about)

     1  package s3manager
     2  
     3  import (
     4  	"bytes"
     5  	"errors"
     6  	"io/ioutil"
     7  	"net/http"
     8  	"net/http/httptest"
     9  	"strings"
    10  	"testing"
    11  
    12  	"github.com/aavshr/aws-sdk-go/aws"
    13  	"github.com/aavshr/aws-sdk-go/aws/awserr"
    14  	"github.com/aavshr/aws-sdk-go/aws/credentials"
    15  	"github.com/aavshr/aws-sdk-go/aws/request"
    16  	"github.com/aavshr/aws-sdk-go/awstesting/unit"
    17  	"github.com/aavshr/aws-sdk-go/service/s3"
    18  	"github.com/aavshr/aws-sdk-go/service/s3/s3iface"
    19  )
    20  
    21  func TestHasParity(t *testing.T) {
    22  	cases := []struct {
    23  		o1       *s3.DeleteObjectsInput
    24  		o2       BatchDeleteObject
    25  		expected bool
    26  	}{
    27  		{
    28  			&s3.DeleteObjectsInput{},
    29  			BatchDeleteObject{
    30  				Object: &s3.DeleteObjectInput{},
    31  			},
    32  			true,
    33  		},
    34  		{
    35  			&s3.DeleteObjectsInput{
    36  				Bucket: aws.String("foo"),
    37  			},
    38  			BatchDeleteObject{
    39  				Object: &s3.DeleteObjectInput{
    40  					Bucket: aws.String("bar"),
    41  				},
    42  			},
    43  			false,
    44  		},
    45  		{
    46  			&s3.DeleteObjectsInput{},
    47  			BatchDeleteObject{
    48  				Object: &s3.DeleteObjectInput{
    49  					Bucket: aws.String("foo"),
    50  				},
    51  			},
    52  			false,
    53  		},
    54  		{
    55  			&s3.DeleteObjectsInput{
    56  				Bucket: aws.String("foo"),
    57  			},
    58  			BatchDeleteObject{
    59  				Object: &s3.DeleteObjectInput{},
    60  			},
    61  			false,
    62  		},
    63  		{
    64  			&s3.DeleteObjectsInput{
    65  				MFA: aws.String("foo"),
    66  			},
    67  			BatchDeleteObject{
    68  				Object: &s3.DeleteObjectInput{
    69  					MFA: aws.String("bar"),
    70  				},
    71  			},
    72  			false,
    73  		},
    74  		{
    75  			&s3.DeleteObjectsInput{},
    76  			BatchDeleteObject{
    77  				Object: &s3.DeleteObjectInput{
    78  					MFA: aws.String("foo"),
    79  				},
    80  			},
    81  			false,
    82  		},
    83  		{
    84  			&s3.DeleteObjectsInput{
    85  				MFA: aws.String("foo"),
    86  			},
    87  			BatchDeleteObject{
    88  				Object: &s3.DeleteObjectInput{},
    89  			},
    90  			false,
    91  		},
    92  		{
    93  			&s3.DeleteObjectsInput{
    94  				RequestPayer: aws.String("foo"),
    95  			},
    96  			BatchDeleteObject{
    97  				Object: &s3.DeleteObjectInput{
    98  					RequestPayer: aws.String("bar"),
    99  				},
   100  			},
   101  			false,
   102  		},
   103  		{
   104  			&s3.DeleteObjectsInput{},
   105  			BatchDeleteObject{
   106  				Object: &s3.DeleteObjectInput{
   107  					RequestPayer: aws.String("foo"),
   108  				},
   109  			},
   110  			false,
   111  		},
   112  		{
   113  			&s3.DeleteObjectsInput{
   114  				RequestPayer: aws.String("foo"),
   115  			},
   116  			BatchDeleteObject{
   117  				Object: &s3.DeleteObjectInput{},
   118  			},
   119  			false,
   120  		},
   121  	}
   122  
   123  	for i, c := range cases {
   124  		if result := hasParity(c.o1, c.o2); result != c.expected {
   125  			t.Errorf("Case %d: expected %t, but received %t\n", i, c.expected, result)
   126  		}
   127  	}
   128  }
   129  
   130  func TestBatchDelete(t *testing.T) {
   131  	cases := []struct {
   132  		objects  []BatchDeleteObject
   133  		size     int
   134  		expected int
   135  	}{
   136  		{
   137  			[]BatchDeleteObject{
   138  				{
   139  					Object: &s3.DeleteObjectInput{
   140  						Key:    aws.String("1"),
   141  						Bucket: aws.String("bucket1"),
   142  					},
   143  				},
   144  				{
   145  					Object: &s3.DeleteObjectInput{
   146  						Key:    aws.String("2"),
   147  						Bucket: aws.String("bucket2"),
   148  					},
   149  				},
   150  				{
   151  					Object: &s3.DeleteObjectInput{
   152  						Key:    aws.String("3"),
   153  						Bucket: aws.String("bucket3"),
   154  					},
   155  				},
   156  				{
   157  					Object: &s3.DeleteObjectInput{
   158  						Key:    aws.String("4"),
   159  						Bucket: aws.String("bucket4"),
   160  					},
   161  				},
   162  			},
   163  			1,
   164  			4,
   165  		},
   166  		{
   167  			[]BatchDeleteObject{
   168  				{
   169  					Object: &s3.DeleteObjectInput{
   170  						Key:    aws.String("1"),
   171  						Bucket: aws.String("bucket1"),
   172  					},
   173  				},
   174  				{
   175  					Object: &s3.DeleteObjectInput{
   176  						Key:    aws.String("2"),
   177  						Bucket: aws.String("bucket1"),
   178  					},
   179  				},
   180  				{
   181  					Object: &s3.DeleteObjectInput{
   182  						Key:    aws.String("3"),
   183  						Bucket: aws.String("bucket3"),
   184  					},
   185  				},
   186  				{
   187  					Object: &s3.DeleteObjectInput{
   188  						Key:    aws.String("4"),
   189  						Bucket: aws.String("bucket3"),
   190  					},
   191  				},
   192  			},
   193  			1,
   194  			4,
   195  		},
   196  		{
   197  			[]BatchDeleteObject{
   198  				{
   199  					Object: &s3.DeleteObjectInput{
   200  						Key:    aws.String("1"),
   201  						Bucket: aws.String("bucket1"),
   202  					},
   203  				},
   204  				{
   205  					Object: &s3.DeleteObjectInput{
   206  						Key:    aws.String("2"),
   207  						Bucket: aws.String("bucket1"),
   208  					},
   209  				},
   210  				{
   211  					Object: &s3.DeleteObjectInput{
   212  						Key:    aws.String("3"),
   213  						Bucket: aws.String("bucket3"),
   214  					},
   215  				},
   216  				{
   217  					Object: &s3.DeleteObjectInput{
   218  						Key:    aws.String("4"),
   219  						Bucket: aws.String("bucket3"),
   220  					},
   221  				},
   222  			},
   223  			4,
   224  			2,
   225  		},
   226  		{
   227  			[]BatchDeleteObject{
   228  				{
   229  					Object: &s3.DeleteObjectInput{
   230  						Key:    aws.String("1"),
   231  						Bucket: aws.String("bucket1"),
   232  					},
   233  				},
   234  				{
   235  					Object: &s3.DeleteObjectInput{
   236  						Key:    aws.String("2"),
   237  						Bucket: aws.String("bucket1"),
   238  					},
   239  				},
   240  				{
   241  					Object: &s3.DeleteObjectInput{
   242  						Key:    aws.String("3"),
   243  						Bucket: aws.String("bucket3"),
   244  					},
   245  				},
   246  				{
   247  					Object: &s3.DeleteObjectInput{
   248  						Key:    aws.String("4"),
   249  						Bucket: aws.String("bucket3"),
   250  					},
   251  				},
   252  			},
   253  			10,
   254  			2,
   255  		},
   256  		{
   257  			[]BatchDeleteObject{
   258  				{
   259  					Object: &s3.DeleteObjectInput{
   260  						Key:    aws.String("1"),
   261  						Bucket: aws.String("bucket1"),
   262  					},
   263  				},
   264  				{
   265  					Object: &s3.DeleteObjectInput{
   266  						Key:    aws.String("2"),
   267  						Bucket: aws.String("bucket1"),
   268  					},
   269  				},
   270  				{
   271  					Object: &s3.DeleteObjectInput{
   272  						Key:    aws.String("3"),
   273  						Bucket: aws.String("bucket1"),
   274  					},
   275  				},
   276  				{
   277  					Object: &s3.DeleteObjectInput{
   278  						Key:    aws.String("4"),
   279  						Bucket: aws.String("bucket3"),
   280  					},
   281  				},
   282  			},
   283  			2,
   284  			3,
   285  		},
   286  	}
   287  
   288  	count := 0
   289  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   290  		w.WriteHeader(http.StatusNoContent)
   291  		count++
   292  	}))
   293  	defer server.Close()
   294  
   295  	svc := &mockS3Client{S3: buildS3SvcClient(server.URL)}
   296  	for i, c := range cases {
   297  		batcher := BatchDelete{
   298  			Client:    svc,
   299  			BatchSize: c.size,
   300  		}
   301  
   302  		if err := batcher.Delete(aws.BackgroundContext(), &DeleteObjectsIterator{Objects: c.objects}); err != nil {
   303  			panic(err)
   304  		}
   305  
   306  		if count != c.expected {
   307  			t.Errorf("Case %d: expected %d, but received %d", i, c.expected, count)
   308  		}
   309  
   310  		count = 0
   311  	}
   312  }
   313  
   314  func TestBatchDeleteError(t *testing.T) {
   315  	cases := []struct {
   316  		objects            []BatchDeleteObject
   317  		output             s3.DeleteObjectsOutput
   318  		size               int
   319  		expectedErrCode    string
   320  		expectedErrMessage string
   321  	}{
   322  		{
   323  			[]BatchDeleteObject{
   324  				{
   325  					Object: &s3.DeleteObjectInput{
   326  						Key:    aws.String("1"),
   327  						Bucket: aws.String("bucket1"),
   328  					},
   329  				},
   330  			},
   331  			s3.DeleteObjectsOutput{
   332  				Errors: []*s3.Error{
   333  					{
   334  						Code:    aws.String("foo code"),
   335  						Message: aws.String("foo error"),
   336  					},
   337  				},
   338  			},
   339  			1,
   340  			"foo code",
   341  			"foo error",
   342  		},
   343  		{
   344  			[]BatchDeleteObject{
   345  				{
   346  					Object: &s3.DeleteObjectInput{
   347  						Key:    aws.String("1"),
   348  						Bucket: aws.String("bucket1"),
   349  					},
   350  				},
   351  			},
   352  			s3.DeleteObjectsOutput{
   353  				Errors: []*s3.Error{
   354  					{},
   355  				},
   356  			},
   357  			1,
   358  			ErrDeleteBatchFailCode,
   359  			errDefaultDeleteBatchMessage,
   360  		},
   361  	}
   362  
   363  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   364  		w.WriteHeader(http.StatusNoContent)
   365  	}))
   366  	defer server.Close()
   367  
   368  	index := 0
   369  	svc := &mockS3Client{
   370  		S3: buildS3SvcClient(server.URL),
   371  		deleteObjects: func(input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) {
   372  			output := &cases[index].output
   373  			index++
   374  			return output, nil
   375  		},
   376  	}
   377  	for _, c := range cases {
   378  		batcher := BatchDelete{
   379  			Client:    svc,
   380  			BatchSize: c.size,
   381  		}
   382  
   383  		err := batcher.Delete(aws.BackgroundContext(), &DeleteObjectsIterator{Objects: c.objects})
   384  		if err == nil {
   385  			t.Errorf("expected error, but received none")
   386  		}
   387  
   388  		berr := err.(*BatchError)
   389  
   390  		if len(berr.Errors) != 1 {
   391  			t.Errorf("expected 1 error, but received %d", len(berr.Errors))
   392  		}
   393  
   394  		aerr := berr.Errors[0].OrigErr.(awserr.Error)
   395  		if e, a := c.expectedErrCode, aerr.Code(); e != a {
   396  			t.Errorf("expected %q, but received %q", e, a)
   397  		}
   398  
   399  		if e, a := c.expectedErrMessage, aerr.Message(); e != a {
   400  			t.Errorf("expected %q, but received %q", e, a)
   401  		}
   402  	}
   403  }
   404  
   405  type mockS3Client struct {
   406  	*s3.S3
   407  	index         int
   408  	objects       []*s3.ListObjectsOutput
   409  	deleteObjects func(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error)
   410  }
   411  
   412  func (client *mockS3Client) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
   413  	object := client.objects[client.index]
   414  	client.index++
   415  	return object, nil
   416  }
   417  
   418  func (client *mockS3Client) DeleteObjects(input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) {
   419  	if client.deleteObjects == nil {
   420  		return client.S3.DeleteObjectsWithContext(aws.BackgroundContext(), input)
   421  	}
   422  
   423  	return client.deleteObjects(input)
   424  }
   425  
   426  func (client *mockS3Client) DeleteObjectsWithContext(ctx aws.Context, input *s3.DeleteObjectsInput, opt ...request.Option) (*s3.DeleteObjectsOutput, error) {
   427  	if client.deleteObjects == nil {
   428  		return client.S3.DeleteObjectsWithContext(ctx, input)
   429  	}
   430  
   431  	return client.deleteObjects(input)
   432  }
   433  
   434  func TestNilOrigError(t *testing.T) {
   435  	err := Error{
   436  		Bucket: aws.String("bucket"),
   437  		Key:    aws.String("key"),
   438  	}
   439  	errStr := err.Error()
   440  	const expected1 = `failed to perform batch operation on "key" to "bucket"`
   441  	if errStr != expected1 {
   442  		t.Errorf("Expected %s, but received %s", expected1, errStr)
   443  	}
   444  
   445  	err = Error{
   446  		OrigErr: errors.New("foo"),
   447  		Bucket:  aws.String("bucket"),
   448  		Key:     aws.String("key"),
   449  	}
   450  	errStr = err.Error()
   451  	const expected2 = "failed to perform batch operation on \"key\" to \"bucket\":\nfoo"
   452  	if errStr != expected2 {
   453  		t.Errorf("Expected %s, but received %s", expected2, errStr)
   454  	}
   455  
   456  }
   457  
   458  func TestBatchDeleteList(t *testing.T) {
   459  	count := 0
   460  
   461  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   462  		w.WriteHeader(http.StatusNoContent)
   463  		count++
   464  	}))
   465  	defer server.Close()
   466  
   467  	objects := []*s3.ListObjectsOutput{
   468  		{
   469  			Contents: []*s3.Object{
   470  				{
   471  					Key: aws.String("1"),
   472  				},
   473  			},
   474  			NextMarker:  aws.String("marker"),
   475  			IsTruncated: aws.Bool(true),
   476  		},
   477  		{
   478  			Contents: []*s3.Object{
   479  				{
   480  					Key: aws.String("2"),
   481  				},
   482  			},
   483  			NextMarker:  aws.String("marker"),
   484  			IsTruncated: aws.Bool(true),
   485  		},
   486  		{
   487  			Contents: []*s3.Object{
   488  				{
   489  					Key: aws.String("3"),
   490  				},
   491  			},
   492  			IsTruncated: aws.Bool(false),
   493  		},
   494  	}
   495  
   496  	svc := &mockS3Client{S3: buildS3SvcClient(server.URL), objects: objects}
   497  	batcher := BatchDelete{
   498  		Client:    svc,
   499  		BatchSize: 1,
   500  	}
   501  
   502  	input := &s3.ListObjectsInput{
   503  		Bucket: aws.String("bucket"),
   504  	}
   505  	iter := &DeleteListIterator{
   506  		Bucket: input.Bucket,
   507  		Paginator: request.Pagination{
   508  			NewRequest: func() (*request.Request, error) {
   509  				var inCpy *s3.ListObjectsInput
   510  				if input != nil {
   511  					tmp := *input
   512  					inCpy = &tmp
   513  				}
   514  				req, _ := svc.ListObjectsRequest(inCpy)
   515  				req.Handlers.Clear()
   516  				output, _ := svc.ListObjects(inCpy)
   517  				req.Data = output
   518  				return req, nil
   519  			},
   520  		},
   521  	}
   522  
   523  	if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil {
   524  		t.Error(err)
   525  	}
   526  
   527  	if count != len(objects) {
   528  		t.Errorf("Expected %d, but received %d", len(objects), count)
   529  	}
   530  }
   531  
   532  func buildS3SvcClient(u string) *s3.S3 {
   533  	return s3.New(unit.Session, &aws.Config{
   534  		Endpoint:         aws.String(u),
   535  		S3ForcePathStyle: aws.Bool(true),
   536  		DisableSSL:       aws.Bool(true),
   537  		Credentials:      credentials.NewStaticCredentials("AKID", "SECRET", "SESSION"),
   538  	})
   539  
   540  }
   541  
   542  func TestBatchDeleteList_EmptyListObjects(t *testing.T) {
   543  	count := 0
   544  
   545  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   546  		w.WriteHeader(http.StatusNoContent)
   547  		count++
   548  	}))
   549  	defer server.Close()
   550  
   551  	svc := &mockS3Client{S3: buildS3SvcClient(server.URL)}
   552  	batcher := BatchDelete{
   553  		Client: svc,
   554  	}
   555  
   556  	input := &s3.ListObjectsInput{
   557  		Bucket: aws.String("bucket"),
   558  	}
   559  
   560  	// Test DeleteListIterator in the case when the ListObjectsRequest responds
   561  	// with an empty listing.
   562  
   563  	// We need a new iterator with a fresh Pagination since
   564  	// Pagination.HasNextPage() is always true the first time Pagination.Next()
   565  	// called on it
   566  	iter := &DeleteListIterator{
   567  		Bucket: input.Bucket,
   568  		Paginator: request.Pagination{
   569  			NewRequest: func() (*request.Request, error) {
   570  				req, _ := svc.ListObjectsRequest(input)
   571  				// Simulate empty listing
   572  				req.Data = &s3.ListObjectsOutput{Contents: []*s3.Object{}}
   573  				return req, nil
   574  			},
   575  		},
   576  	}
   577  
   578  	if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil {
   579  		t.Error(err)
   580  	}
   581  	if count != 1 {
   582  		t.Errorf("expect count to be 1, got %d", count)
   583  	}
   584  }
   585  
   586  func TestBatchDownload(t *testing.T) {
   587  	count := 0
   588  	expected := []struct {
   589  		bucket, key string
   590  	}{
   591  		{
   592  			key:    "1",
   593  			bucket: "bucket1",
   594  		},
   595  		{
   596  			key:    "2",
   597  			bucket: "bucket2",
   598  		},
   599  		{
   600  			key:    "3",
   601  			bucket: "bucket3",
   602  		},
   603  		{
   604  			key:    "4",
   605  			bucket: "bucket4",
   606  		},
   607  	}
   608  
   609  	received := []struct {
   610  		bucket, key string
   611  	}{}
   612  
   613  	payload := []string{
   614  		"1",
   615  		"2",
   616  		"3",
   617  		"4",
   618  	}
   619  
   620  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   621  		urlParts := strings.Split(r.URL.String(), "/")
   622  		received = append(received, struct{ bucket, key string }{urlParts[1], urlParts[2]})
   623  		w.Write([]byte(payload[count]))
   624  		count++
   625  	}))
   626  	defer server.Close()
   627  
   628  	svc := NewDownloaderWithClient(buildS3SvcClient(server.URL))
   629  
   630  	objects := []BatchDownloadObject{
   631  		{
   632  			Object: &s3.GetObjectInput{
   633  				Key:    aws.String("1"),
   634  				Bucket: aws.String("bucket1"),
   635  			},
   636  			Writer: aws.NewWriteAtBuffer(make([]byte, 128)),
   637  		},
   638  		{
   639  			Object: &s3.GetObjectInput{
   640  				Key:    aws.String("2"),
   641  				Bucket: aws.String("bucket2"),
   642  			},
   643  			Writer: aws.NewWriteAtBuffer(make([]byte, 128)),
   644  		},
   645  		{
   646  			Object: &s3.GetObjectInput{
   647  				Key:    aws.String("3"),
   648  				Bucket: aws.String("bucket3"),
   649  			},
   650  			Writer: aws.NewWriteAtBuffer(make([]byte, 128)),
   651  		},
   652  		{
   653  			Object: &s3.GetObjectInput{
   654  				Key:    aws.String("4"),
   655  				Bucket: aws.String("bucket4"),
   656  			},
   657  			Writer: aws.NewWriteAtBuffer(make([]byte, 128)),
   658  		},
   659  	}
   660  
   661  	iter := &DownloadObjectsIterator{Objects: objects}
   662  	if err := svc.DownloadWithIterator(aws.BackgroundContext(), iter); err != nil {
   663  		panic(err)
   664  	}
   665  
   666  	if count != len(objects) {
   667  		t.Errorf("Expected %d, but received %d", len(objects), count)
   668  	}
   669  
   670  	if len(expected) != len(received) {
   671  		t.Errorf("Expected %d, but received %d", len(expected), len(received))
   672  	}
   673  
   674  	for i := 0; i < len(expected); i++ {
   675  		if expected[i].key != received[i].key {
   676  			t.Errorf("Expected %q, but received %q", expected[i].key, received[i].key)
   677  		}
   678  
   679  		if expected[i].bucket != received[i].bucket {
   680  			t.Errorf("Expected %q, but received %q", expected[i].bucket, received[i].bucket)
   681  		}
   682  	}
   683  
   684  	for i, p := range payload {
   685  		b := iter.Objects[i].Writer.(*aws.WriteAtBuffer).Bytes()
   686  		b = bytes.Trim(b, "\x00")
   687  
   688  		if string(b) != p {
   689  			t.Errorf("Expected %q, but received %q", p, b)
   690  		}
   691  	}
   692  }
   693  
   694  func TestBatchUpload(t *testing.T) {
   695  	count := 0
   696  	expected := []struct {
   697  		bucket, key string
   698  		reqBody     string
   699  	}{
   700  		{
   701  			key:     "1",
   702  			bucket:  "bucket1",
   703  			reqBody: "1",
   704  		},
   705  		{
   706  			key:     "2",
   707  			bucket:  "bucket2",
   708  			reqBody: "2",
   709  		},
   710  		{
   711  			key:     "3",
   712  			bucket:  "bucket3",
   713  			reqBody: "3",
   714  		},
   715  		{
   716  			key:     "4",
   717  			bucket:  "bucket4",
   718  			reqBody: "4",
   719  		},
   720  	}
   721  
   722  	received := []struct {
   723  		bucket, key, reqBody string
   724  	}{}
   725  
   726  	payload := []string{
   727  		"a",
   728  		"b",
   729  		"c",
   730  		"d",
   731  	}
   732  
   733  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   734  		urlParts := strings.Split(r.URL.String(), "/")
   735  
   736  		b, err := ioutil.ReadAll(r.Body)
   737  		if err != nil {
   738  			t.Error(err)
   739  		}
   740  
   741  		received = append(received, struct{ bucket, key, reqBody string }{urlParts[1], urlParts[2], string(b)})
   742  		w.Write([]byte(payload[count]))
   743  
   744  		count++
   745  	}))
   746  	defer server.Close()
   747  
   748  	svc := NewUploaderWithClient(buildS3SvcClient(server.URL))
   749  
   750  	objects := []BatchUploadObject{
   751  		{
   752  			Object: &UploadInput{
   753  				Key:    aws.String("1"),
   754  				Bucket: aws.String("bucket1"),
   755  				Body:   bytes.NewBuffer([]byte("1")),
   756  			},
   757  		},
   758  		{
   759  			Object: &UploadInput{
   760  				Key:    aws.String("2"),
   761  				Bucket: aws.String("bucket2"),
   762  				Body:   bytes.NewBuffer([]byte("2")),
   763  			},
   764  		},
   765  		{
   766  			Object: &UploadInput{
   767  				Key:    aws.String("3"),
   768  				Bucket: aws.String("bucket3"),
   769  				Body:   bytes.NewBuffer([]byte("3")),
   770  			},
   771  		},
   772  		{
   773  			Object: &UploadInput{
   774  				Key:    aws.String("4"),
   775  				Bucket: aws.String("bucket4"),
   776  				Body:   bytes.NewBuffer([]byte("4")),
   777  			},
   778  		},
   779  	}
   780  
   781  	iter := &UploadObjectsIterator{Objects: objects}
   782  	if err := svc.UploadWithIterator(aws.BackgroundContext(), iter); err != nil {
   783  		panic(err)
   784  	}
   785  
   786  	if count != len(objects) {
   787  		t.Errorf("Expected %d, but received %d", len(objects), count)
   788  	}
   789  
   790  	if len(expected) != len(received) {
   791  		t.Errorf("Expected %d, but received %d", len(expected), len(received))
   792  	}
   793  
   794  	for i := 0; i < len(expected); i++ {
   795  		if expected[i].key != received[i].key {
   796  			t.Errorf("Expected %q, but received %q", expected[i].key, received[i].key)
   797  		}
   798  
   799  		if expected[i].bucket != received[i].bucket {
   800  			t.Errorf("Expected %q, but received %q", expected[i].bucket, received[i].bucket)
   801  		}
   802  
   803  		if expected[i].reqBody != received[i].reqBody {
   804  			t.Errorf("Expected %q, but received %q", expected[i].reqBody, received[i].reqBody)
   805  		}
   806  	}
   807  }
   808  
   809  type mockClient struct {
   810  	s3iface.S3API
   811  	Put       func() (*s3.PutObjectOutput, error)
   812  	Get       func() (*s3.GetObjectOutput, error)
   813  	List      func() (*s3.ListObjectsOutput, error)
   814  	responses []response
   815  }
   816  
   817  type response struct {
   818  	out interface{}
   819  	err error
   820  }
   821  
   822  func (client *mockClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
   823  	return client.Put()
   824  }
   825  
   826  func (client *mockClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) {
   827  	req, _ := client.S3API.PutObjectRequest(input)
   828  	req.Handlers.Clear()
   829  	req.Data, req.Error = client.Put()
   830  	return req, req.Data.(*s3.PutObjectOutput)
   831  }
   832  
   833  func (client *mockClient) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
   834  	return client.List()
   835  }
   836  
   837  func (client *mockClient) ListObjectsRequest(input *s3.ListObjectsInput) (*request.Request, *s3.ListObjectsOutput) {
   838  	req, _ := client.S3API.ListObjectsRequest(input)
   839  	req.Handlers.Clear()
   840  	req.Data, req.Error = client.List()
   841  	return req, req.Data.(*s3.ListObjectsOutput)
   842  }
   843  
   844  func TestBatchError(t *testing.T) {
   845  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   846  	}))
   847  	defer server.Close()
   848  
   849  	index := 0
   850  	responses := []response{
   851  		{
   852  			&s3.PutObjectOutput{},
   853  			errors.New("Foo"),
   854  		},
   855  		{
   856  			&s3.PutObjectOutput{},
   857  			nil,
   858  		},
   859  		{
   860  			&s3.PutObjectOutput{},
   861  			nil,
   862  		},
   863  		{
   864  			&s3.PutObjectOutput{},
   865  			errors.New("Bar"),
   866  		},
   867  	}
   868  
   869  	svc := &mockClient{
   870  		S3API: buildS3SvcClient(server.URL),
   871  		Put: func() (*s3.PutObjectOutput, error) {
   872  			resp := responses[index]
   873  			index++
   874  			return resp.out.(*s3.PutObjectOutput), resp.err
   875  		},
   876  		List: func() (*s3.ListObjectsOutput, error) {
   877  			resp := responses[index]
   878  			index++
   879  			return resp.out.(*s3.ListObjectsOutput), resp.err
   880  		},
   881  	}
   882  	uploader := NewUploaderWithClient(svc)
   883  
   884  	objects := []BatchUploadObject{
   885  		{
   886  			Object: &UploadInput{
   887  				Key:    aws.String("1"),
   888  				Bucket: aws.String("bucket1"),
   889  				Body:   bytes.NewBuffer([]byte("1")),
   890  			},
   891  		},
   892  		{
   893  			Object: &UploadInput{
   894  				Key:    aws.String("2"),
   895  				Bucket: aws.String("bucket2"),
   896  				Body:   bytes.NewBuffer([]byte("2")),
   897  			},
   898  		},
   899  		{
   900  			Object: &UploadInput{
   901  				Key:    aws.String("3"),
   902  				Bucket: aws.String("bucket3"),
   903  				Body:   bytes.NewBuffer([]byte("3")),
   904  			},
   905  		},
   906  		{
   907  			Object: &UploadInput{
   908  				Key:    aws.String("4"),
   909  				Bucket: aws.String("bucket4"),
   910  				Body:   bytes.NewBuffer([]byte("4")),
   911  			},
   912  		},
   913  	}
   914  
   915  	iter := &UploadObjectsIterator{Objects: objects}
   916  	if err := uploader.UploadWithIterator(aws.BackgroundContext(), iter); err != nil {
   917  		if bErr, ok := err.(*BatchError); !ok {
   918  			t.Error("Expected BatchError, but received other")
   919  		} else {
   920  			if len(bErr.Errors) != 2 {
   921  				t.Errorf("Expected 2 errors, but received %d", len(bErr.Errors))
   922  			}
   923  
   924  			expected := []struct {
   925  				bucket, key string
   926  			}{
   927  				{
   928  					"bucket1",
   929  					"1",
   930  				},
   931  				{
   932  					"bucket4",
   933  					"4",
   934  				},
   935  			}
   936  			for i, expect := range expected {
   937  				if *bErr.Errors[i].Bucket != expect.bucket {
   938  					t.Errorf("Case %d: Invalid bucket expected %s, but received %s", i, expect.bucket, *bErr.Errors[i].Bucket)
   939  				}
   940  
   941  				if *bErr.Errors[i].Key != expect.key {
   942  					t.Errorf("Case %d: Invalid key expected %s, but received %s", i, expect.key, *bErr.Errors[i].Key)
   943  				}
   944  			}
   945  		}
   946  	} else {
   947  		t.Error("Expected error, but received nil")
   948  	}
   949  
   950  	if index != len(objects) {
   951  		t.Errorf("Expected %d, but received %d", len(objects), index)
   952  	}
   953  
   954  }
   955  
   956  type testAfterDeleteIter struct {
   957  	afterDelete   bool
   958  	afterDownload bool
   959  	afterUpload   bool
   960  	next          bool
   961  }
   962  
   963  func (iter *testAfterDeleteIter) Next() bool {
   964  	next := !iter.next
   965  	iter.next = !iter.next
   966  	return next
   967  }
   968  
   969  func (iter *testAfterDeleteIter) Err() error {
   970  	return nil
   971  }
   972  
   973  func (iter *testAfterDeleteIter) DeleteObject() BatchDeleteObject {
   974  	return BatchDeleteObject{
   975  		Object: &s3.DeleteObjectInput{
   976  			Bucket: aws.String("foo"),
   977  			Key:    aws.String("foo"),
   978  		},
   979  		After: func() error {
   980  			iter.afterDelete = true
   981  			return nil
   982  		},
   983  	}
   984  }
   985  
   986  type testAfterDownloadIter struct {
   987  	afterDownload bool
   988  	afterUpload   bool
   989  	next          bool
   990  }
   991  
   992  func (iter *testAfterDownloadIter) Next() bool {
   993  	next := !iter.next
   994  	iter.next = !iter.next
   995  	return next
   996  }
   997  
   998  func (iter *testAfterDownloadIter) Err() error {
   999  	return nil
  1000  }
  1001  
  1002  func (iter *testAfterDownloadIter) DownloadObject() BatchDownloadObject {
  1003  	return BatchDownloadObject{
  1004  		Object: &s3.GetObjectInput{
  1005  			Bucket: aws.String("foo"),
  1006  			Key:    aws.String("foo"),
  1007  		},
  1008  		Writer: aws.NewWriteAtBuffer([]byte{}),
  1009  		After: func() error {
  1010  			iter.afterDownload = true
  1011  			return nil
  1012  		},
  1013  	}
  1014  }
  1015  
  1016  type testAfterUploadIter struct {
  1017  	afterUpload bool
  1018  	next        bool
  1019  }
  1020  
  1021  func (iter *testAfterUploadIter) Next() bool {
  1022  	next := !iter.next
  1023  	iter.next = !iter.next
  1024  	return next
  1025  }
  1026  
  1027  func (iter *testAfterUploadIter) Err() error {
  1028  	return nil
  1029  }
  1030  
  1031  func (iter *testAfterUploadIter) UploadObject() BatchUploadObject {
  1032  	return BatchUploadObject{
  1033  		Object: &UploadInput{
  1034  			Bucket: aws.String("foo"),
  1035  			Key:    aws.String("foo"),
  1036  			Body:   strings.NewReader("bar"),
  1037  		},
  1038  		After: func() error {
  1039  			iter.afterUpload = true
  1040  			return nil
  1041  		},
  1042  	}
  1043  }
  1044  
  1045  func TestAfter(t *testing.T) {
  1046  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1047  	}))
  1048  	defer server.Close()
  1049  
  1050  	index := 0
  1051  	responses := []response{
  1052  		{
  1053  			&s3.PutObjectOutput{},
  1054  			nil,
  1055  		},
  1056  		{
  1057  			&s3.GetObjectOutput{},
  1058  			nil,
  1059  		},
  1060  		{
  1061  			&s3.DeleteObjectOutput{},
  1062  			nil,
  1063  		},
  1064  	}
  1065  
  1066  	svc := &mockClient{
  1067  		S3API: buildS3SvcClient(server.URL),
  1068  		Put: func() (*s3.PutObjectOutput, error) {
  1069  			resp := responses[index]
  1070  			index++
  1071  			return resp.out.(*s3.PutObjectOutput), resp.err
  1072  		},
  1073  		Get: func() (*s3.GetObjectOutput, error) {
  1074  			resp := responses[index]
  1075  			index++
  1076  			return resp.out.(*s3.GetObjectOutput), resp.err
  1077  		},
  1078  		List: func() (*s3.ListObjectsOutput, error) {
  1079  			resp := responses[index]
  1080  			index++
  1081  			return resp.out.(*s3.ListObjectsOutput), resp.err
  1082  		},
  1083  	}
  1084  	uploader := NewUploaderWithClient(svc)
  1085  	downloader := NewDownloaderWithClient(svc)
  1086  	deleter := NewBatchDeleteWithClient(svc)
  1087  
  1088  	deleteIter := &testAfterDeleteIter{}
  1089  	downloadIter := &testAfterDownloadIter{}
  1090  	uploadIter := &testAfterUploadIter{}
  1091  
  1092  	if err := uploader.UploadWithIterator(aws.BackgroundContext(), uploadIter); err != nil {
  1093  		t.Error(err)
  1094  	}
  1095  
  1096  	if err := downloader.DownloadWithIterator(aws.BackgroundContext(), downloadIter); err != nil {
  1097  		t.Error(err)
  1098  	}
  1099  
  1100  	if err := deleter.Delete(aws.BackgroundContext(), deleteIter); err != nil {
  1101  		t.Error(err)
  1102  	}
  1103  
  1104  	if !deleteIter.afterDelete {
  1105  		t.Error("Expected 'afterDelete' to be true, but received false")
  1106  	}
  1107  
  1108  	if !downloadIter.afterDownload {
  1109  		t.Error("Expected 'afterDownload' to be true, but received false")
  1110  	}
  1111  
  1112  	if !uploadIter.afterUpload {
  1113  		t.Error("Expected 'afterUpload' to be true, but received false")
  1114  	}
  1115  }