github.com/lnzx/goofys@v0.24.0/internal/backend_s3.go (about) 1 // Copyright 2019 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 . "github.com/kahing/goofys/api/common" 19 20 "fmt" 21 "net/http" 22 "net/url" 23 "strconv" 24 "strings" 25 "sync/atomic" 26 "syscall" 27 "time" 28 29 "github.com/aws/aws-sdk-go/aws" 30 "github.com/aws/aws-sdk-go/aws/awserr" 31 "github.com/aws/aws-sdk-go/aws/corehandlers" 32 "github.com/aws/aws-sdk-go/aws/credentials" 33 "github.com/aws/aws-sdk-go/aws/request" 34 "github.com/aws/aws-sdk-go/service/s3" 35 36 "github.com/jacobsa/fuse" 37 ) 38 39 type S3Backend struct { 40 *s3.S3 41 cap Capabilities 42 43 bucket string 44 awsConfig *aws.Config 45 flags *FlagStorage 46 config *S3Config 47 sseType string 48 49 aws bool 50 gcs bool 51 v2Signer bool 52 } 53 54 func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, error) { 55 awsConfig, err := config.ToAwsConfig(flags) 56 if err != nil { 57 return nil, err 58 } 59 s := &S3Backend{ 60 bucket: bucket, 61 awsConfig: awsConfig, 62 flags: flags, 63 config: config, 64 cap: Capabilities{ 65 Name: "s3", 66 MaxMultipartSize: 5 * 1024 * 1024 * 1024, 67 }, 68 } 69 70 if flags.DebugS3 { 71 awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors) 72 } 73 74 if config.UseKMS { 75 //SSE header string for KMS server-side encryption (SSE-KMS) 76 s.sseType = s3.ServerSideEncryptionAwsKms 77 } else if config.UseSSE { 78 //SSE header string for non-KMS server-side encryption (SSE-S3) 79 s.sseType = s3.ServerSideEncryptionAes256 80 } 81 82 s.newS3() 83 return s, nil 84 } 85 86 func (s *S3Backend) Bucket() string { 87 return s.bucket 88 } 89 90 func (s *S3Backend) Capabilities() *Capabilities { 91 return &s.cap 92 } 93 94 func addAcceptEncoding(req *request.Request) { 95 if req.HTTPRequest.Method == "GET" { 96 // we need "Accept-Encoding: identity" so that objects 97 // with content-encoding won't be automatically 98 // deflated, but we don't want to sign it because GCS 99 // doesn't like it 100 req.HTTPRequest.Header.Set("Accept-Encoding", "identity") 101 } 102 } 103 104 func addRequestPayer(req *request.Request) { 105 // "Requester Pays" is only applicable to these 106 // see https://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html 107 if req.HTTPRequest.Method == "GET" || req.HTTPRequest.Method == "HEAD" || req.HTTPRequest.Method == "POST" { 108 req.HTTPRequest.Header.Set("x-amz-request-payer", "requester") 109 } 110 } 111 112 func (s *S3Backend) setV2Signer(handlers *request.Handlers) { 113 handlers.Sign.Clear() 114 handlers.Sign.PushBack(SignV2) 115 handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) 116 } 117 118 func (s *S3Backend) newS3() { 119 s.S3 = s3.New(s.config.Session, s.awsConfig) 120 if s.config.RequesterPays { 121 s.S3.Handlers.Build.PushBack(addRequestPayer) 122 } 123 if s.v2Signer { 124 s.setV2Signer(&s.S3.Handlers) 125 } 126 s.S3.Handlers.Sign.PushBack(addAcceptEncoding) 127 } 128 129 func (s *S3Backend) detectBucketLocationByHEAD() (err error, isAws bool) { 130 u := url.URL{ 131 Scheme: "https", 132 Host: "s3.amazonaws.com", 133 Path: s.bucket, 134 } 135 136 if s.awsConfig.Endpoint != nil { 137 endpoint, err := url.Parse(*s.awsConfig.Endpoint) 138 if err != nil { 139 return err, false 140 } 141 142 u.Scheme = endpoint.Scheme 143 u.Host = endpoint.Host 144 } 145 146 var req *http.Request 147 var resp *http.Response 148 149 req, err = http.NewRequest("HEAD", u.String(), nil) 150 if err != nil { 151 return 152 } 153 154 allowFails := 3 155 for i := 0; i < allowFails; i++ { 156 resp, err = http.DefaultTransport.RoundTrip(req) 157 if err != nil { 158 return 159 } 160 if resp.StatusCode < 500 { 161 break 162 } else if resp.StatusCode == 503 && resp.Status == "503 Slow Down" { 163 time.Sleep(time.Duration(i+1) * time.Second) 164 // allow infinite retries for 503 slow down 165 allowFails += 1 166 } 167 } 168 169 region := resp.Header["X-Amz-Bucket-Region"] 170 server := resp.Header["Server"] 171 172 s3Log.Debugf("HEAD %v = %v %v", u.String(), resp.StatusCode, region) 173 if region == nil { 174 for k, v := range resp.Header { 175 s3Log.Debugf("%v = %v", k, v) 176 } 177 } 178 if server != nil && server[0] == "AmazonS3" { 179 isAws = true 180 } 181 182 switch resp.StatusCode { 183 case 200: 184 // note that this only happen if the bucket is in us-east-1 185 if len(s.config.Profile) == 0 { 186 s.awsConfig.Credentials = credentials.AnonymousCredentials 187 s3Log.Infof("anonymous bucket detected") 188 } 189 case 400: 190 err = fuse.EINVAL 191 case 403: 192 err = syscall.EACCES 193 case 404: 194 err = syscall.ENXIO 195 case 405: 196 err = syscall.ENOTSUP 197 default: 198 err = awserr.New(strconv.Itoa(resp.StatusCode), resp.Status, nil) 199 } 200 201 if len(region) != 0 { 202 if region[0] != *s.awsConfig.Region { 203 s3Log.Infof("Switching from region '%v' to '%v'", 204 *s.awsConfig.Region, region[0]) 205 s.awsConfig.Region = ®ion[0] 206 } 207 208 // we detected a region, this is aws, the error is irrelevant 209 err = nil 210 } 211 212 return 213 } 214 215 func (s *S3Backend) testBucket(key string) (err error) { 216 _, err = s.HeadBlob(&HeadBlobInput{Key: key}) 217 if err != nil { 218 if err == fuse.ENOENT { 219 err = nil 220 } 221 } 222 223 return 224 } 225 226 func (s *S3Backend) fallbackV2Signer() (err error) { 227 if s.v2Signer { 228 return fuse.EINVAL 229 } 230 231 s3Log.Infoln("Falling back to v2 signer") 232 s.v2Signer = true 233 s.newS3() 234 return 235 } 236 237 func (s *S3Backend) Init(key string) error { 238 var isAws bool 239 var err error 240 241 if !s.config.RegionSet { 242 err, isAws = s.detectBucketLocationByHEAD() 243 if err == nil { 244 // we detected a region header, this is probably AWS S3, 245 // or we can use anonymous access, or both 246 s.newS3() 247 s.aws = isAws 248 } else if err == syscall.ENXIO { 249 return fmt.Errorf("bucket %v does not exist", s.bucket) 250 } else { 251 // this is NOT AWS, we expect the request to fail with 403 if this is not 252 // an anonymous bucket 253 if err != syscall.EACCES { 254 s3Log.Errorf("Unable to access '%v': %v", s.bucket, err) 255 } 256 } 257 } 258 259 // try again with the credential to make sure 260 err = s.testBucket(key) 261 if err != nil { 262 if !isAws { 263 // EMC returns 403 because it doesn't support v4 signing 264 // swift3, ceph-s3 returns 400 265 // Amplidata just gives up and return 500 266 if err == syscall.EACCES || err == fuse.EINVAL || err == syscall.EAGAIN { 267 err = s.fallbackV2Signer() 268 if err != nil { 269 return err 270 } 271 err = s.testBucket(key) 272 } 273 } 274 275 if err != nil { 276 return err 277 } 278 } 279 280 return nil 281 } 282 283 func (s *S3Backend) ListObjectsV2(params *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, string, error) { 284 if s.aws { 285 req, resp := s.S3.ListObjectsV2Request(params) 286 err := req.Send() 287 if err != nil { 288 return nil, "", err 289 } 290 return resp, s.getRequestId(req), nil 291 } else { 292 v1 := s3.ListObjectsInput{ 293 Bucket: params.Bucket, 294 Delimiter: params.Delimiter, 295 EncodingType: params.EncodingType, 296 MaxKeys: params.MaxKeys, 297 Prefix: params.Prefix, 298 RequestPayer: params.RequestPayer, 299 } 300 if params.StartAfter != nil { 301 v1.Marker = params.StartAfter 302 } else { 303 v1.Marker = params.ContinuationToken 304 } 305 306 objs, err := s.S3.ListObjects(&v1) 307 if err != nil { 308 return nil, "", err 309 } 310 311 count := int64(len(objs.Contents)) 312 v2Objs := s3.ListObjectsV2Output{ 313 CommonPrefixes: objs.CommonPrefixes, 314 Contents: objs.Contents, 315 ContinuationToken: objs.Marker, 316 Delimiter: objs.Delimiter, 317 EncodingType: objs.EncodingType, 318 IsTruncated: objs.IsTruncated, 319 KeyCount: &count, 320 MaxKeys: objs.MaxKeys, 321 Name: objs.Name, 322 NextContinuationToken: objs.NextMarker, 323 Prefix: objs.Prefix, 324 StartAfter: objs.Marker, 325 } 326 327 return &v2Objs, "", nil 328 } 329 } 330 331 func metadataToLower(m map[string]*string) map[string]*string { 332 if m != nil { 333 var toDelete []string 334 for k, v := range m { 335 lower := strings.ToLower(k) 336 if lower != k { 337 m[lower] = v 338 toDelete = append(toDelete, k) 339 } 340 } 341 for _, k := range toDelete { 342 delete(m, k) 343 } 344 } 345 return m 346 } 347 348 func (s *S3Backend) getRequestId(r *request.Request) string { 349 return r.HTTPResponse.Header.Get("x-amz-request-id") + ": " + 350 r.HTTPResponse.Header.Get("x-amz-id-2") 351 } 352 353 func (s *S3Backend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 354 head := s3.HeadObjectInput{Bucket: &s.bucket, 355 Key: ¶m.Key, 356 } 357 if s.config.SseC != "" { 358 head.SSECustomerAlgorithm = PString("AES256") 359 head.SSECustomerKey = &s.config.SseC 360 head.SSECustomerKeyMD5 = &s.config.SseCDigest 361 } 362 363 req, resp := s.S3.HeadObjectRequest(&head) 364 err := req.Send() 365 if err != nil { 366 return nil, mapAwsError(err) 367 } 368 return &HeadBlobOutput{ 369 BlobItemOutput: BlobItemOutput{ 370 Key: ¶m.Key, 371 ETag: resp.ETag, 372 LastModified: resp.LastModified, 373 Size: uint64(*resp.ContentLength), 374 StorageClass: resp.StorageClass, 375 }, 376 ContentType: resp.ContentType, 377 Metadata: metadataToLower(resp.Metadata), 378 IsDirBlob: strings.HasSuffix(param.Key, "/"), 379 RequestId: s.getRequestId(req), 380 }, nil 381 } 382 383 func (s *S3Backend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 384 var maxKeys *int64 385 386 if param.MaxKeys != nil { 387 maxKeys = aws.Int64(int64(*param.MaxKeys)) 388 } 389 390 resp, reqId, err := s.ListObjectsV2(&s3.ListObjectsV2Input{ 391 Bucket: &s.bucket, 392 Prefix: param.Prefix, 393 Delimiter: param.Delimiter, 394 MaxKeys: maxKeys, 395 StartAfter: param.StartAfter, 396 ContinuationToken: param.ContinuationToken, 397 }) 398 if err != nil { 399 return nil, mapAwsError(err) 400 } 401 402 prefixes := make([]BlobPrefixOutput, 0) 403 items := make([]BlobItemOutput, 0) 404 405 for _, p := range resp.CommonPrefixes { 406 prefixes = append(prefixes, BlobPrefixOutput{Prefix: p.Prefix}) 407 } 408 for _, i := range resp.Contents { 409 items = append(items, BlobItemOutput{ 410 Key: i.Key, 411 ETag: i.ETag, 412 LastModified: i.LastModified, 413 Size: uint64(*i.Size), 414 StorageClass: i.StorageClass, 415 }) 416 } 417 418 return &ListBlobsOutput{ 419 Prefixes: prefixes, 420 Items: items, 421 NextContinuationToken: resp.NextContinuationToken, 422 IsTruncated: *resp.IsTruncated, 423 RequestId: reqId, 424 }, nil 425 } 426 427 func (s *S3Backend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 428 req, _ := s.DeleteObjectRequest(&s3.DeleteObjectInput{ 429 Bucket: &s.bucket, 430 Key: ¶m.Key, 431 }) 432 err := req.Send() 433 if err != nil { 434 return nil, mapAwsError(err) 435 } 436 return &DeleteBlobOutput{s.getRequestId(req)}, nil 437 } 438 439 func (s *S3Backend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 440 num_objs := len(param.Items) 441 442 var items s3.Delete 443 var objs = make([]*s3.ObjectIdentifier, num_objs) 444 445 for i, _ := range param.Items { 446 objs[i] = &s3.ObjectIdentifier{Key: ¶m.Items[i]} 447 } 448 449 // Add list of objects to delete to Delete object 450 items.SetObjects(objs) 451 452 req, _ := s.DeleteObjectsRequest(&s3.DeleteObjectsInput{ 453 Bucket: &s.bucket, 454 Delete: &items, 455 }) 456 err := req.Send() 457 if err != nil { 458 return nil, mapAwsError(err) 459 } 460 461 return &DeleteBlobsOutput{s.getRequestId(req)}, nil 462 } 463 464 func (s *S3Backend) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 465 return nil, syscall.ENOTSUP 466 } 467 468 func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes string, part int64, 469 sem semaphore, srcEtag *string, etag **string, errout *error) { 470 471 defer sem.P(1) 472 473 // XXX use CopySourceIfUnmodifiedSince to ensure that 474 // we are copying from the same object 475 params := &s3.UploadPartCopyInput{ 476 Bucket: &s.bucket, 477 Key: &to, 478 CopySource: aws.String(pathEscape(from)), 479 UploadId: &mpuId, 480 CopySourceRange: &bytes, 481 CopySourceIfMatch: srcEtag, 482 PartNumber: &part, 483 } 484 if s.config.SseC != "" { 485 params.SSECustomerAlgorithm = PString("AES256") 486 params.SSECustomerKey = &s.config.SseC 487 params.SSECustomerKeyMD5 = &s.config.SseCDigest 488 params.CopySourceSSECustomerAlgorithm = PString("AES256") 489 params.CopySourceSSECustomerKey = &s.config.SseC 490 params.CopySourceSSECustomerKeyMD5 = &s.config.SseCDigest 491 } 492 493 s3Log.Debug(params) 494 495 resp, err := s.UploadPartCopy(params) 496 if err != nil { 497 s3Log.Errorf("UploadPartCopy %v = %v", params, err) 498 *errout = mapAwsError(err) 499 return 500 } 501 502 *etag = resp.CopyPartResult.ETag 503 return 504 } 505 506 func sizeToParts(size int64) (int, int64) { 507 const MAX_S3_MPU_SIZE = 5 * 1024 * 1024 * 1024 * 1024 508 if size > MAX_S3_MPU_SIZE { 509 panic(fmt.Sprintf("object size: %v exceeds maximum S3 MPU size: %v", size, MAX_S3_MPU_SIZE)) 510 } 511 512 // Use the maximum number of parts to allow the most server-side copy 513 // parallelism. 514 const MAX_PARTS = 10 * 1000 515 const MIN_PART_SIZE = 50 * 1024 * 1024 516 partSize := MaxInt64(size/(MAX_PARTS-1), MIN_PART_SIZE) 517 518 nParts := int(size / partSize) 519 if size%partSize != 0 { 520 nParts++ 521 } 522 523 return nParts, partSize 524 } 525 526 func (s *S3Backend) mpuCopyParts(size int64, from string, to string, mpuId string, 527 srcEtag *string, etags []*string, partSize int64, err *error) { 528 529 rangeFrom := int64(0) 530 rangeTo := int64(0) 531 532 MAX_CONCURRENCY := MinInt(100, len(etags)) 533 sem := make(semaphore, MAX_CONCURRENCY) 534 sem.P(MAX_CONCURRENCY) 535 536 for i := int64(1); rangeTo < size; i++ { 537 rangeFrom = rangeTo 538 rangeTo = i * partSize 539 if rangeTo > size { 540 rangeTo = size 541 } 542 bytes := fmt.Sprintf("bytes=%v-%v", rangeFrom, rangeTo-1) 543 544 sem.V(1) 545 go s.mpuCopyPart(from, to, mpuId, bytes, i, sem, srcEtag, &etags[i-1], err) 546 } 547 548 sem.V(MAX_CONCURRENCY) 549 } 550 551 func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuId string, 552 srcEtag *string, metadata map[string]*string, storageClass *string) (requestId string, err error) { 553 nParts, partSize := sizeToParts(size) 554 etags := make([]*string, nParts) 555 556 if mpuId == "" { 557 params := &s3.CreateMultipartUploadInput{ 558 Bucket: &s.bucket, 559 Key: &to, 560 StorageClass: storageClass, 561 ContentType: s.flags.GetMimeType(to), 562 Metadata: metadataToLower(metadata), 563 } 564 565 if s.config.UseSSE { 566 params.ServerSideEncryption = &s.sseType 567 if s.config.UseKMS && s.config.KMSKeyID != "" { 568 params.SSEKMSKeyId = &s.config.KMSKeyID 569 } 570 } else if s.config.SseC != "" { 571 params.SSECustomerAlgorithm = PString("AES256") 572 params.SSECustomerKey = &s.config.SseC 573 params.SSECustomerKeyMD5 = &s.config.SseCDigest 574 } 575 576 if s.config.ACL != "" { 577 params.ACL = &s.config.ACL 578 } 579 580 resp, err := s.CreateMultipartUpload(params) 581 if err != nil { 582 return "", mapAwsError(err) 583 } 584 585 mpuId = *resp.UploadId 586 } 587 588 s.mpuCopyParts(size, from, to, mpuId, srcEtag, etags, partSize, &err) 589 590 if err != nil { 591 return 592 } else { 593 parts := make([]*s3.CompletedPart, nParts) 594 for i := 0; i < nParts; i++ { 595 parts[i] = &s3.CompletedPart{ 596 ETag: etags[i], 597 PartNumber: aws.Int64(int64(i + 1)), 598 } 599 } 600 601 params := &s3.CompleteMultipartUploadInput{ 602 Bucket: &s.bucket, 603 Key: &to, 604 UploadId: &mpuId, 605 MultipartUpload: &s3.CompletedMultipartUpload{ 606 Parts: parts, 607 }, 608 } 609 610 s3Log.Debug(params) 611 612 req, _ := s.CompleteMultipartUploadRequest(params) 613 err = req.Send() 614 if err != nil { 615 s3Log.Errorf("Complete MPU %v = %v", params, err) 616 err = mapAwsError(err) 617 } else { 618 requestId = s.getRequestId(req) 619 } 620 } 621 622 return 623 } 624 625 func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 626 metadataDirective := s3.MetadataDirectiveCopy 627 if param.Metadata != nil { 628 metadataDirective = s3.MetadataDirectiveReplace 629 } 630 631 COPY_LIMIT := uint64(5 * 1024 * 1024 * 1024) 632 633 if param.Size == nil || param.ETag == nil || (*param.Size > COPY_LIMIT && 634 (param.Metadata == nil || param.StorageClass == nil)) { 635 636 params := &HeadBlobInput{Key: param.Source} 637 resp, err := s.HeadBlob(params) 638 if err != nil { 639 return nil, err 640 } 641 642 param.Size = &resp.Size 643 param.ETag = resp.ETag 644 if param.Metadata == nil { 645 param.Metadata = resp.Metadata 646 } 647 param.StorageClass = resp.StorageClass 648 } 649 650 if param.StorageClass == nil { 651 if *param.Size < 128*1024 && s.config.StorageClass == "STANDARD_IA" { 652 param.StorageClass = PString("STANDARD") 653 } else { 654 param.StorageClass = &s.config.StorageClass 655 } 656 } 657 658 from := s.bucket + "/" + param.Source 659 660 if !s.gcs && *param.Size > COPY_LIMIT { 661 reqId, err := s.copyObjectMultipart(int64(*param.Size), from, param.Destination, "", param.ETag, param.Metadata, param.StorageClass) 662 if err != nil { 663 return nil, err 664 } 665 return &CopyBlobOutput{reqId}, nil 666 } 667 668 params := &s3.CopyObjectInput{ 669 Bucket: &s.bucket, 670 CopySource: aws.String(pathEscape(from)), 671 Key: ¶m.Destination, 672 StorageClass: param.StorageClass, 673 ContentType: s.flags.GetMimeType(param.Destination), 674 Metadata: metadataToLower(param.Metadata), 675 MetadataDirective: &metadataDirective, 676 } 677 678 s3Log.Debug(params) 679 680 if s.config.UseSSE { 681 params.ServerSideEncryption = &s.sseType 682 if s.config.UseKMS && s.config.KMSKeyID != "" { 683 params.SSEKMSKeyId = &s.config.KMSKeyID 684 } 685 } else if s.config.SseC != "" { 686 params.SSECustomerAlgorithm = PString("AES256") 687 params.SSECustomerKey = &s.config.SseC 688 params.SSECustomerKeyMD5 = &s.config.SseCDigest 689 params.CopySourceSSECustomerAlgorithm = PString("AES256") 690 params.CopySourceSSECustomerKey = &s.config.SseC 691 params.CopySourceSSECustomerKeyMD5 = &s.config.SseCDigest 692 } 693 694 if s.config.ACL != "" { 695 params.ACL = &s.config.ACL 696 } 697 698 req, _ := s.CopyObjectRequest(params) 699 // make a shallow copy of the client so we can change the 700 // timeout only for this request but still re-use the 701 // connection pool 702 c := *(req.Config.HTTPClient) 703 req.Config.HTTPClient = &c 704 req.Config.HTTPClient.Timeout = 15 * time.Minute 705 err := req.Send() 706 if err != nil { 707 s3Log.Errorf("CopyObject %v = %v", params, err) 708 return nil, mapAwsError(err) 709 } 710 711 return &CopyBlobOutput{s.getRequestId(req)}, nil 712 } 713 714 func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 715 get := s3.GetObjectInput{ 716 Bucket: &s.bucket, 717 Key: ¶m.Key, 718 } 719 720 if s.config.SseC != "" { 721 get.SSECustomerAlgorithm = PString("AES256") 722 get.SSECustomerKey = &s.config.SseC 723 get.SSECustomerKeyMD5 = &s.config.SseCDigest 724 } 725 726 if param.Start != 0 || param.Count != 0 { 727 var bytes string 728 if param.Count != 0 { 729 bytes = fmt.Sprintf("bytes=%v-%v", param.Start, param.Start+param.Count-1) 730 } else { 731 bytes = fmt.Sprintf("bytes=%v-", param.Start) 732 } 733 get.Range = &bytes 734 } 735 // TODO handle IfMatch 736 737 req, resp := s.GetObjectRequest(&get) 738 err := req.Send() 739 if err != nil { 740 return nil, mapAwsError(err) 741 } 742 743 return &GetBlobOutput{ 744 HeadBlobOutput: HeadBlobOutput{ 745 BlobItemOutput: BlobItemOutput{ 746 Key: ¶m.Key, 747 ETag: resp.ETag, 748 LastModified: resp.LastModified, 749 Size: uint64(*resp.ContentLength), 750 StorageClass: resp.StorageClass, 751 }, 752 ContentType: resp.ContentType, 753 Metadata: metadataToLower(resp.Metadata), 754 }, 755 Body: resp.Body, 756 RequestId: s.getRequestId(req), 757 }, nil 758 } 759 760 func getDate(resp *http.Response) *time.Time { 761 date := resp.Header.Get("Date") 762 if date != "" { 763 t, err := http.ParseTime(date) 764 if err == nil { 765 return &t 766 } 767 s3Log.Warnf("invalidate date for %v: %v", 768 resp.Request.URL.Path, date) 769 } 770 return nil 771 } 772 773 func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 774 storageClass := s.config.StorageClass 775 if param.Size != nil && *param.Size < 128*1024 && storageClass == "STANDARD_IA" { 776 storageClass = "STANDARD" 777 } 778 779 put := &s3.PutObjectInput{ 780 Bucket: &s.bucket, 781 Key: ¶m.Key, 782 Metadata: metadataToLower(param.Metadata), 783 Body: param.Body, 784 StorageClass: &storageClass, 785 ContentType: param.ContentType, 786 } 787 788 if s.config.UseSSE { 789 put.ServerSideEncryption = &s.sseType 790 if s.config.UseKMS && s.config.KMSKeyID != "" { 791 put.SSEKMSKeyId = &s.config.KMSKeyID 792 } 793 } else if s.config.SseC != "" { 794 put.SSECustomerAlgorithm = PString("AES256") 795 put.SSECustomerKey = &s.config.SseC 796 put.SSECustomerKeyMD5 = &s.config.SseCDigest 797 } 798 799 if s.config.ACL != "" { 800 put.ACL = &s.config.ACL 801 } 802 803 req, resp := s.PutObjectRequest(put) 804 err := req.Send() 805 if err != nil { 806 return nil, mapAwsError(err) 807 } 808 809 return &PutBlobOutput{ 810 ETag: resp.ETag, 811 LastModified: getDate(req.HTTPResponse), 812 StorageClass: &storageClass, 813 RequestId: s.getRequestId(req), 814 }, nil 815 } 816 817 func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 818 mpu := s3.CreateMultipartUploadInput{ 819 Bucket: &s.bucket, 820 Key: ¶m.Key, 821 StorageClass: &s.config.StorageClass, 822 ContentType: param.ContentType, 823 } 824 825 if s.config.UseSSE { 826 mpu.ServerSideEncryption = &s.sseType 827 if s.config.UseKMS && s.config.KMSKeyID != "" { 828 mpu.SSEKMSKeyId = &s.config.KMSKeyID 829 } 830 } else if s.config.SseC != "" { 831 mpu.SSECustomerAlgorithm = PString("AES256") 832 mpu.SSECustomerKey = &s.config.SseC 833 mpu.SSECustomerKeyMD5 = &s.config.SseCDigest 834 } 835 836 if s.config.ACL != "" { 837 mpu.ACL = &s.config.ACL 838 } 839 840 resp, err := s.CreateMultipartUpload(&mpu) 841 if err != nil { 842 s3Log.Errorf("CreateMultipartUpload %v = %v", param.Key, err) 843 return nil, mapAwsError(err) 844 } 845 846 return &MultipartBlobCommitInput{ 847 Key: ¶m.Key, 848 Metadata: metadataToLower(param.Metadata), 849 UploadId: resp.UploadId, 850 Parts: make([]*string, 10000), // at most 10K parts 851 }, nil 852 } 853 854 func (s *S3Backend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 855 en := ¶m.Commit.Parts[param.PartNumber-1] 856 atomic.AddUint32(¶m.Commit.NumParts, 1) 857 858 params := s3.UploadPartInput{ 859 Bucket: &s.bucket, 860 Key: param.Commit.Key, 861 PartNumber: aws.Int64(int64(param.PartNumber)), 862 UploadId: param.Commit.UploadId, 863 Body: param.Body, 864 } 865 if s.config.SseC != "" { 866 params.SSECustomerAlgorithm = PString("AES256") 867 params.SSECustomerKey = &s.config.SseC 868 params.SSECustomerKeyMD5 = &s.config.SseCDigest 869 } 870 s3Log.Debug(params) 871 872 req, resp := s.UploadPartRequest(¶ms) 873 err := req.Send() 874 if err != nil { 875 return nil, mapAwsError(err) 876 } 877 878 if *en != nil { 879 panic(fmt.Sprintf("etags for part %v already set: %v", param.PartNumber, **en)) 880 } 881 *en = resp.ETag 882 883 return &MultipartBlobAddOutput{s.getRequestId(req)}, nil 884 } 885 886 func (s *S3Backend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 887 parts := make([]*s3.CompletedPart, param.NumParts) 888 for i := uint32(0); i < param.NumParts; i++ { 889 parts[i] = &s3.CompletedPart{ 890 ETag: param.Parts[i], 891 PartNumber: aws.Int64(int64(i + 1)), 892 } 893 } 894 895 mpu := s3.CompleteMultipartUploadInput{ 896 Bucket: &s.bucket, 897 Key: param.Key, 898 UploadId: param.UploadId, 899 MultipartUpload: &s3.CompletedMultipartUpload{ 900 Parts: parts, 901 }, 902 } 903 904 s3Log.Debug(mpu) 905 906 req, resp := s.CompleteMultipartUploadRequest(&mpu) 907 err := req.Send() 908 if err != nil { 909 return nil, mapAwsError(err) 910 } 911 912 s3Log.Debug(resp) 913 914 return &MultipartBlobCommitOutput{ 915 ETag: resp.ETag, 916 LastModified: getDate(req.HTTPResponse), 917 RequestId: s.getRequestId(req), 918 }, nil 919 } 920 921 func (s *S3Backend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 922 mpu := s3.AbortMultipartUploadInput{ 923 Bucket: &s.bucket, 924 Key: param.Key, 925 UploadId: param.UploadId, 926 } 927 req, _ := s.AbortMultipartUploadRequest(&mpu) 928 err := req.Send() 929 if err != nil { 930 return nil, mapAwsError(err) 931 } 932 return &MultipartBlobAbortOutput{s.getRequestId(req)}, nil 933 } 934 935 func (s *S3Backend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 936 mpu, err := s.ListMultipartUploads(&s3.ListMultipartUploadsInput{ 937 Bucket: &s.bucket, 938 }) 939 if err != nil { 940 return nil, mapAwsError(err) 941 } 942 s3Log.Debug(mpu) 943 944 now := time.Now() 945 for _, upload := range mpu.Uploads { 946 expireTime := upload.Initiated.Add(48 * time.Hour) 947 948 if !expireTime.After(now) { 949 params := &s3.AbortMultipartUploadInput{ 950 Bucket: &s.bucket, 951 Key: upload.Key, 952 UploadId: upload.UploadId, 953 } 954 resp, err := s.AbortMultipartUpload(params) 955 s3Log.Debug(resp) 956 957 if mapAwsError(err) == syscall.EACCES { 958 break 959 } 960 } else { 961 s3Log.Debugf("Keeping MPU Key=%v Id=%v", *upload.Key, *upload.UploadId) 962 } 963 } 964 965 return &MultipartExpireOutput{}, nil 966 } 967 968 func (s *S3Backend) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 969 _, err := s.DeleteBucket(&s3.DeleteBucketInput{Bucket: &s.bucket}) 970 if err != nil { 971 return nil, mapAwsError(err) 972 } 973 return &RemoveBucketOutput{}, nil 974 } 975 976 func (s *S3Backend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 977 _, err := s.CreateBucket(&s3.CreateBucketInput{ 978 Bucket: &s.bucket, 979 ACL: &s.config.ACL, 980 }) 981 if err != nil { 982 return nil, mapAwsError(err) 983 } 984 985 if s.config.BucketOwner != "" { 986 var owner s3.Tag 987 owner.SetKey("Owner") 988 owner.SetValue(s.config.BucketOwner) 989 990 param := s3.PutBucketTaggingInput{ 991 Bucket: &s.bucket, 992 Tagging: &s3.Tagging{ 993 TagSet: []*s3.Tag{&owner}, 994 }, 995 } 996 997 for i := 0; i < 10; i++ { 998 _, err = s.PutBucketTagging(¶m) 999 err = mapAwsError((err)) 1000 switch err { 1001 case nil: 1002 break 1003 case syscall.ENXIO, syscall.EINTR: 1004 s3Log.Infof("waiting for bucket") 1005 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 1006 default: 1007 s3Log.Errorf("Failed to tag bucket %v: %v", s.bucket, err) 1008 return nil, err 1009 } 1010 } 1011 } 1012 1013 return &MakeBucketOutput{}, err 1014 } 1015 1016 func (s *S3Backend) Delegate() interface{} { 1017 return s 1018 }