github.com/swisspost/terratest@v0.0.0-20230214120104-7ec6de2e1ae0/modules/aws/s3.go (about) 1 package aws 2 3 import ( 4 "bytes" 5 "fmt" 6 "strings" 7 8 "github.com/aws/aws-sdk-go/aws" 9 "github.com/aws/aws-sdk-go/service/s3" 10 "github.com/aws/aws-sdk-go/service/s3/s3manager" 11 "github.com/gruntwork-io/terratest/modules/logger" 12 "github.com/gruntwork-io/terratest/modules/testing" 13 "github.com/stretchr/testify/require" 14 ) 15 16 // FindS3BucketWithTag finds the name of the S3 bucket in the given region with the given tag key=value. 17 func FindS3BucketWithTag(t testing.TestingT, awsRegion string, key string, value string) string { 18 bucket, err := FindS3BucketWithTagE(t, awsRegion, key, value) 19 require.NoError(t, err) 20 21 return bucket 22 } 23 24 // FindS3BucketWithTagE finds the name of the S3 bucket in the given region with the given tag key=value. 25 func FindS3BucketWithTagE(t testing.TestingT, awsRegion string, key string, value string) (string, error) { 26 s3Client, err := NewS3ClientE(t, awsRegion) 27 if err != nil { 28 return "", err 29 } 30 31 resp, err := s3Client.ListBuckets(&s3.ListBucketsInput{}) 32 if err != nil { 33 return "", err 34 } 35 36 for _, bucket := range resp.Buckets { 37 tagResponse, err := s3Client.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: bucket.Name}) 38 39 if err != nil { 40 if strings.Contains(err.Error(), "NoSuchBucket") { 41 // Occasionally, the ListBuckets call will return a bucket that has been deleted by S3 42 // but hasn't yet been actually removed from the backend. Listing tags on that bucket 43 // will return this error. If the bucket has been deleted, it can't be the one to find, 44 // so just ignore this error, and keep checking the other buckets. 45 continue 46 } 47 if !strings.Contains(err.Error(), "AuthorizationHeaderMalformed") && 48 !strings.Contains(err.Error(), "BucketRegionError") && 49 !strings.Contains(err.Error(), "NoSuchTagSet") { 50 return "", err 51 } 52 } 53 54 for _, tag := range tagResponse.TagSet { 55 if *tag.Key == key && *tag.Value == value { 56 logger.Logf(t, "Found S3 bucket %s with tag %s=%s", *bucket.Name, key, value) 57 return *bucket.Name, nil 58 } 59 } 60 } 61 62 return "", nil 63 } 64 65 // GetS3BucketTags fetches the given bucket's tags and returns them as a string map of strings. 66 func GetS3BucketTags(t testing.TestingT, awsRegion string, bucket string) map[string]string { 67 tags, err := GetS3BucketTagsE(t, awsRegion, bucket) 68 require.NoError(t, err) 69 70 return tags 71 } 72 73 // GetS3BucketTagsE fetches the given bucket's tags and returns them as a string map of strings. 74 func GetS3BucketTagsE(t testing.TestingT, awsRegion string, bucket string) (map[string]string, error) { 75 s3Client, err := NewS3ClientE(t, awsRegion) 76 if err != nil { 77 return nil, err 78 } 79 80 out, err := s3Client.GetBucketTagging(&s3.GetBucketTaggingInput{ 81 Bucket: &bucket, 82 }) 83 if err != nil { 84 return nil, err 85 } 86 87 tags := map[string]string{} 88 for _, tag := range out.TagSet { 89 tags[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value) 90 } 91 92 return tags, nil 93 } 94 95 // GetS3ObjectContents fetches the contents of the object in the given bucket with the given key and return it as a string. 96 func GetS3ObjectContents(t testing.TestingT, awsRegion string, bucket string, key string) string { 97 contents, err := GetS3ObjectContentsE(t, awsRegion, bucket, key) 98 require.NoError(t, err) 99 100 return contents 101 } 102 103 // GetS3ObjectContentsE fetches the contents of the object in the given bucket with the given key and return it as a string. 104 func GetS3ObjectContentsE(t testing.TestingT, awsRegion string, bucket string, key string) (string, error) { 105 s3Client, err := NewS3ClientE(t, awsRegion) 106 if err != nil { 107 return "", err 108 } 109 110 res, err := s3Client.GetObject(&s3.GetObjectInput{ 111 Bucket: &bucket, 112 Key: &key, 113 }) 114 115 if err != nil { 116 return "", err 117 } 118 119 buf := new(bytes.Buffer) 120 _, err = buf.ReadFrom(res.Body) 121 if err != nil { 122 return "", err 123 } 124 125 contents := buf.String() 126 logger.Logf(t, "Read contents from s3://%s/%s", bucket, key) 127 128 return contents, nil 129 } 130 131 // CreateS3Bucket creates an S3 bucket in the given region with the given name. Note that S3 bucket names must be globally unique. 132 func CreateS3Bucket(t testing.TestingT, region string, name string) { 133 err := CreateS3BucketE(t, region, name) 134 require.NoError(t, err) 135 } 136 137 // CreateS3BucketE creates an S3 bucket in the given region with the given name. Note that S3 bucket names must be globally unique. 138 func CreateS3BucketE(t testing.TestingT, region string, name string) error { 139 logger.Logf(t, "Creating bucket %s in %s", name, region) 140 141 s3Client, err := NewS3ClientE(t, region) 142 if err != nil { 143 return err 144 } 145 146 params := &s3.CreateBucketInput{ 147 Bucket: aws.String(name), 148 } 149 _, err = s3Client.CreateBucket(params) 150 return err 151 } 152 153 // PutS3BucketPolicy applies an IAM resource policy to a given S3 bucket to create it's bucket policy 154 func PutS3BucketPolicy(t testing.TestingT, region string, bucketName string, policyJSONString string) { 155 err := PutS3BucketPolicyE(t, region, bucketName, policyJSONString) 156 require.NoError(t, err) 157 } 158 159 // PutS3BucketPolicyE applies an IAM resource policy to a given S3 bucket to create it's bucket policy 160 func PutS3BucketPolicyE(t testing.TestingT, region string, bucketName string, policyJSONString string) error { 161 logger.Logf(t, "Applying bucket policy for bucket %s in %s", bucketName, region) 162 163 s3Client, err := NewS3ClientE(t, region) 164 if err != nil { 165 return err 166 } 167 168 input := &s3.PutBucketPolicyInput{ 169 Bucket: aws.String(bucketName), 170 Policy: aws.String(policyJSONString), 171 } 172 173 _, err = s3Client.PutBucketPolicy(input) 174 return err 175 } 176 177 // PutS3BucketVersioning creates an S3 bucket versioning configuration in the given region against the given bucket name, WITHOUT requiring MFA to remove versioning. 178 func PutS3BucketVersioning(t testing.TestingT, region string, bucketName string) { 179 err := PutS3BucketVersioningE(t, region, bucketName) 180 require.NoError(t, err) 181 } 182 183 // PutS3BucketVersioningE creates an S3 bucket versioning configuration in the given region against the given bucket name, WITHOUT requiring MFA to remove versioning. 184 func PutS3BucketVersioningE(t testing.TestingT, region string, bucketName string) error { 185 logger.Logf(t, "Creating bucket versioning configuration for bucket %s in %s", bucketName, region) 186 187 s3Client, err := NewS3ClientE(t, region) 188 if err != nil { 189 return err 190 } 191 192 input := &s3.PutBucketVersioningInput{ 193 Bucket: aws.String(bucketName), 194 VersioningConfiguration: &s3.VersioningConfiguration{ 195 MFADelete: aws.String("Disabled"), 196 Status: aws.String("Enabled"), 197 }, 198 } 199 200 _, err = s3Client.PutBucketVersioning(input) 201 return err 202 } 203 204 // DeleteS3Bucket destroys the S3 bucket in the given region with the given name. 205 func DeleteS3Bucket(t testing.TestingT, region string, name string) { 206 err := DeleteS3BucketE(t, region, name) 207 require.NoError(t, err) 208 } 209 210 // DeleteS3BucketE destroys the S3 bucket in the given region with the given name. 211 func DeleteS3BucketE(t testing.TestingT, region string, name string) error { 212 logger.Logf(t, "Deleting bucket %s in %s", region, name) 213 214 s3Client, err := NewS3ClientE(t, region) 215 if err != nil { 216 return err 217 } 218 219 params := &s3.DeleteBucketInput{ 220 Bucket: aws.String(name), 221 } 222 _, err = s3Client.DeleteBucket(params) 223 return err 224 } 225 226 // EmptyS3Bucket removes the contents of an S3 bucket in the given region with the given name. 227 func EmptyS3Bucket(t testing.TestingT, region string, name string) { 228 err := EmptyS3BucketE(t, region, name) 229 require.NoError(t, err) 230 } 231 232 // EmptyS3BucketE removes the contents of an S3 bucket in the given region with the given name. 233 func EmptyS3BucketE(t testing.TestingT, region string, name string) error { 234 logger.Logf(t, "Emptying bucket %s in %s", name, region) 235 236 s3Client, err := NewS3ClientE(t, region) 237 if err != nil { 238 return err 239 } 240 241 params := &s3.ListObjectVersionsInput{ 242 Bucket: aws.String(name), 243 } 244 245 for { 246 // Requesting a batch of objects from s3 bucket 247 bucketObjects, err := s3Client.ListObjectVersions(params) 248 if err != nil { 249 return err 250 } 251 252 //Checks if the bucket is already empty 253 if len((*bucketObjects).Versions) == 0 { 254 logger.Logf(t, "Bucket %s is already empty", name) 255 return nil 256 } 257 258 //creating an array of pointers of ObjectIdentifier 259 objectsToDelete := make([]*s3.ObjectIdentifier, 0, 1000) 260 for _, object := range (*bucketObjects).Versions { 261 obj := s3.ObjectIdentifier{ 262 Key: object.Key, 263 VersionId: object.VersionId, 264 } 265 objectsToDelete = append(objectsToDelete, &obj) 266 } 267 268 for _, object := range (*bucketObjects).DeleteMarkers { 269 obj := s3.ObjectIdentifier{ 270 Key: object.Key, 271 VersionId: object.VersionId, 272 } 273 objectsToDelete = append(objectsToDelete, &obj) 274 } 275 276 //Creating JSON payload for bulk delete 277 deleteArray := s3.Delete{Objects: objectsToDelete} 278 deleteParams := &s3.DeleteObjectsInput{ 279 Bucket: aws.String(name), 280 Delete: &deleteArray, 281 } 282 283 //Running the Bulk delete job (limit 1000) 284 _, err = s3Client.DeleteObjects(deleteParams) 285 if err != nil { 286 return err 287 } 288 289 if *(*bucketObjects).IsTruncated { //if there are more objects in the bucket, IsTruncated = true 290 // params.Marker = (*deleteParams).Delete.Objects[len((*deleteParams).Delete.Objects)-1].Key 291 params.KeyMarker = bucketObjects.NextKeyMarker 292 logger.Logf(t, "Requesting next batch | %s", *(params.KeyMarker)) 293 } else { //if all objects in the bucket have been cleaned up. 294 break 295 } 296 } 297 logger.Logf(t, "Bucket %s is now empty", name) 298 return err 299 } 300 301 // GetS3BucketLoggingTarget fetches the given bucket's logging target bucket and returns it as a string 302 func GetS3BucketLoggingTarget(t testing.TestingT, awsRegion string, bucket string) string { 303 loggingTarget, err := GetS3BucketLoggingTargetE(t, awsRegion, bucket) 304 require.NoError(t, err) 305 306 return loggingTarget 307 } 308 309 // GetS3BucketLoggingTargetE fetches the given bucket's logging target bucket and returns it as the following string: 310 // `TargetBucket` of the `LoggingEnabled` property for an S3 bucket 311 func GetS3BucketLoggingTargetE(t testing.TestingT, awsRegion string, bucket string) (string, error) { 312 s3Client, err := NewS3ClientE(t, awsRegion) 313 if err != nil { 314 return "", err 315 } 316 317 res, err := s3Client.GetBucketLogging(&s3.GetBucketLoggingInput{ 318 Bucket: &bucket, 319 }) 320 321 if err != nil { 322 return "", err 323 } 324 325 if res.LoggingEnabled == nil { 326 return "", S3AccessLoggingNotEnabledErr{bucket, awsRegion} 327 } 328 329 return aws.StringValue(res.LoggingEnabled.TargetBucket), nil 330 } 331 332 // GetS3BucketLoggingTargetPrefix fetches the given bucket's logging object prefix and returns it as a string 333 func GetS3BucketLoggingTargetPrefix(t testing.TestingT, awsRegion string, bucket string) string { 334 loggingObjectTargetPrefix, err := GetS3BucketLoggingTargetPrefixE(t, awsRegion, bucket) 335 require.NoError(t, err) 336 337 return loggingObjectTargetPrefix 338 } 339 340 // GetS3BucketLoggingTargetPrefixE fetches the given bucket's logging object prefix and returns it as the following string: 341 // `TargetPrefix` of the `LoggingEnabled` property for an S3 bucket 342 func GetS3BucketLoggingTargetPrefixE(t testing.TestingT, awsRegion string, bucket string) (string, error) { 343 s3Client, err := NewS3ClientE(t, awsRegion) 344 if err != nil { 345 return "", err 346 } 347 348 res, err := s3Client.GetBucketLogging(&s3.GetBucketLoggingInput{ 349 Bucket: &bucket, 350 }) 351 352 if err != nil { 353 return "", err 354 } 355 356 if res.LoggingEnabled == nil { 357 return "", S3AccessLoggingNotEnabledErr{bucket, awsRegion} 358 } 359 360 return aws.StringValue(res.LoggingEnabled.TargetPrefix), nil 361 } 362 363 // GetS3BucketVersioning fetches the given bucket's versioning configuration status and returns it as a string 364 func GetS3BucketVersioning(t testing.TestingT, awsRegion string, bucket string) string { 365 versioningStatus, err := GetS3BucketVersioningE(t, awsRegion, bucket) 366 require.NoError(t, err) 367 368 return versioningStatus 369 } 370 371 // GetS3BucketVersioningE fetches the given bucket's versioning configuration status and returns it as a string 372 func GetS3BucketVersioningE(t testing.TestingT, awsRegion string, bucket string) (string, error) { 373 s3Client, err := NewS3ClientE(t, awsRegion) 374 if err != nil { 375 return "", err 376 } 377 378 res, err := s3Client.GetBucketVersioning(&s3.GetBucketVersioningInput{ 379 Bucket: &bucket, 380 }) 381 if err != nil { 382 return "", err 383 } 384 385 return aws.StringValue(res.Status), nil 386 } 387 388 // GetS3BucketPolicy fetches the given bucket's resource policy and returns it as a string 389 func GetS3BucketPolicy(t testing.TestingT, awsRegion string, bucket string) string { 390 bucketPolicy, err := GetS3BucketPolicyE(t, awsRegion, bucket) 391 require.NoError(t, err) 392 393 return bucketPolicy 394 } 395 396 // GetS3BucketPolicyE fetches the given bucket's resource policy and returns it as a string 397 func GetS3BucketPolicyE(t testing.TestingT, awsRegion string, bucket string) (string, error) { 398 s3Client, err := NewS3ClientE(t, awsRegion) 399 if err != nil { 400 return "", err 401 } 402 403 res, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{ 404 Bucket: &bucket, 405 }) 406 if err != nil { 407 return "", err 408 } 409 410 return aws.StringValue(res.Policy), nil 411 } 412 413 // AssertS3BucketExists checks if the given S3 bucket exists in the given region and fail the test if it does not. 414 func AssertS3BucketExists(t testing.TestingT, region string, name string) { 415 err := AssertS3BucketExistsE(t, region, name) 416 require.NoError(t, err) 417 } 418 419 // AssertS3BucketExistsE checks if the given S3 bucket exists in the given region and return an error if it does not. 420 func AssertS3BucketExistsE(t testing.TestingT, region string, name string) error { 421 s3Client, err := NewS3ClientE(t, region) 422 if err != nil { 423 return err 424 } 425 426 params := &s3.HeadBucketInput{ 427 Bucket: aws.String(name), 428 } 429 _, err = s3Client.HeadBucket(params) 430 return err 431 } 432 433 // AssertS3BucketVersioningExists checks if the given S3 bucket has a versioning configuration enabled and returns an error if it does not. 434 func AssertS3BucketVersioningExists(t testing.TestingT, region string, bucketName string) { 435 err := AssertS3BucketVersioningExistsE(t, region, bucketName) 436 require.NoError(t, err) 437 } 438 439 // AssertS3BucketVersioningExistsE checks if the given S3 bucket has a versioning configuration enabled and returns an error if it does not. 440 func AssertS3BucketVersioningExistsE(t testing.TestingT, region string, bucketName string) error { 441 status, err := GetS3BucketVersioningE(t, region, bucketName) 442 if err != nil { 443 return err 444 } 445 446 if status == "Enabled" { 447 return nil 448 } 449 return NewBucketVersioningNotEnabledError(bucketName, region, status) 450 } 451 452 // AssertS3BucketPolicyExists checks if the given S3 bucket has a resource policy attached and returns an error if it does not 453 func AssertS3BucketPolicyExists(t testing.TestingT, region string, bucketName string) { 454 err := AssertS3BucketPolicyExistsE(t, region, bucketName) 455 require.NoError(t, err) 456 } 457 458 // AssertS3BucketPolicyExistsE checks if the given S3 bucket has a resource policy attached and returns an error if it does not 459 func AssertS3BucketPolicyExistsE(t testing.TestingT, region string, bucketName string) error { 460 policy, err := GetS3BucketPolicyE(t, region, bucketName) 461 if err != nil { 462 return err 463 } 464 465 if policy == "" { 466 return NewNoBucketPolicyError(bucketName, region, policy) 467 } 468 return nil 469 } 470 471 // NewS3Client creates an S3 client. 472 func NewS3Client(t testing.TestingT, region string) *s3.S3 { 473 client, err := NewS3ClientE(t, region) 474 require.NoError(t, err) 475 476 return client 477 } 478 479 // NewS3ClientE creates an S3 client. 480 func NewS3ClientE(t testing.TestingT, region string) (*s3.S3, error) { 481 sess, err := NewAuthenticatedSession(region) 482 if err != nil { 483 return nil, err 484 } 485 486 return s3.New(sess), nil 487 } 488 489 // NewS3Uploader creates an S3 Uploader. 490 func NewS3Uploader(t testing.TestingT, region string) *s3manager.Uploader { 491 uploader, err := NewS3UploaderE(t, region) 492 require.NoError(t, err) 493 return uploader 494 } 495 496 // NewS3UploaderE creates an S3 Uploader. 497 func NewS3UploaderE(t testing.TestingT, region string) (*s3manager.Uploader, error) { 498 sess, err := NewAuthenticatedSession(region) 499 if err != nil { 500 return nil, err 501 } 502 503 return s3manager.NewUploader(sess), nil 504 } 505 506 // S3AccessLoggingNotEnabledErr is a custom error that occurs when acess logging hasn't been enabled on the S3 Bucket 507 type S3AccessLoggingNotEnabledErr struct { 508 OriginBucket string 509 Region string 510 } 511 512 func (err S3AccessLoggingNotEnabledErr) Error() string { 513 return fmt.Sprintf("Server Acess Logging hasn't been enabled for S3 Bucket %s in region %s", err.OriginBucket, err.Region) 514 }