github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/chat/s3/s3.go (about) 1 // 2 // goamz - Go packages to interact with the Amazon Web Services. 3 // 4 // https://wiki.ubuntu.com/goamz 5 // 6 // Copyright (c) 2011 Canonical Ltd. 7 // 8 // Written by Gustavo Niemeyer <gustavo.niemeyer@canonical.com> 9 // 10 // Modified by Keybase to allow external signing of requests. 11 12 package s3 13 14 import ( 15 "bytes" 16 "crypto/md5" 17 "encoding/base64" 18 "encoding/xml" 19 "fmt" 20 "io" 21 "log" 22 "net" 23 "net/http" 24 "net/http/httputil" 25 "net/url" 26 "strconv" 27 "strings" 28 "time" 29 30 "github.com/keybase/client/go/libkb" 31 32 "golang.org/x/net/context" 33 ) 34 35 const debug = false 36 37 type Signer interface { 38 Sign(payload []byte) ([]byte, error) 39 } 40 41 // The S3 type encapsulates operations with an S3 region. 42 type S3 struct { 43 Region 44 libkb.Contextified 45 46 // This is needed for payload construction. It's 47 // ok for clients to know it. 48 AccessKey string 49 50 // This is needed for payload construction. It's 51 // ok for clients to know it. 52 SessionToken string 53 54 // Signer signs payloads for s3 request authorization. 55 Signer Signer 56 57 // ConnectTimeout is the maximum time a request attempt will 58 // wait for a successful connection to be made. 59 // 60 // A value of zero means no timeout. 61 ConnectTimeout time.Duration 62 63 // ReadTimeout is the maximum time a request attempt will wait 64 // for an individual read to complete. 65 // 66 // A value of zero means no timeout. 67 ReadTimeout time.Duration 68 69 // WriteTimeout is the maximum time a request attempt will 70 // wait for an individual write to complete. 71 // 72 // A value of zero means no timeout. 73 WriteTimeout time.Duration 74 75 // RequestTimeout is the maximum time a request attempt can 76 // take before operations return a timeout error. 77 // 78 // This includes connection time, any redirects, and reading 79 // the response body. The timer remains running after the request 80 // is made so it can interrupt reading of the response data. 81 // 82 // A Timeout of zero means no timeout. 83 RequestTimeout time.Duration 84 85 // AttemptStrategy is the attempt strategy used for requests. 86 AttemptStrategy 87 88 // client used for requests 89 client *http.Client 90 } 91 92 // The Bucket type encapsulates operations with an S3 bucket. 93 type Bucket struct { 94 *S3 95 Name string 96 } 97 98 // The Owner type represents the owner of the object in an S3 bucket. 99 type Owner struct { 100 ID string 101 DisplayName string 102 } 103 104 // Fold options into an Options struct 105 type Options struct { 106 SSE bool 107 Meta map[string][]string 108 ContentEncoding string 109 CacheControl string 110 RedirectLocation string 111 ContentMD5 string 112 // What else? 113 // Content-Disposition string 114 //// The following become headers so they are []strings rather than strings... I think 115 // x-amz-storage-class []string 116 } 117 118 type CopyOptions struct { 119 Options 120 MetadataDirective string 121 ContentType string 122 } 123 124 // CopyObjectResult is the output from a Copy request 125 type CopyObjectResult struct { 126 ETag string 127 LastModified string 128 } 129 130 // DefaultAttemptStrategy is the default AttemptStrategy used by S3 objects created by New. 131 var DefaultAttemptStrategy = AttemptStrategy{ 132 Min: 5, 133 Total: 5 * time.Second, 134 Delay: 200 * time.Millisecond, 135 } 136 137 // New creates a new S3. Optional client argument allows for custom http.clients to be used. 138 func New(g *libkb.GlobalContext, signer Signer, region Region, client ...*http.Client) *S3 { 139 140 var httpclient *http.Client 141 142 if len(client) > 0 { 143 httpclient = client[0] 144 } 145 146 return &S3{ 147 Signer: signer, 148 Region: region, 149 AttemptStrategy: DefaultAttemptStrategy, 150 client: httpclient, 151 Contextified: libkb.NewContextified(g), 152 } 153 } 154 155 func (s3 *S3) SetAccessKey(key string) { 156 s3.AccessKey = key 157 } 158 159 func (s3 *S3) SetSessionToken(token string) { 160 s3.SessionToken = token 161 } 162 163 // Bucket returns a Bucket with the given name. 164 func (s3 *S3) Bucket(name string) BucketInt { 165 if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket { 166 name = strings.ToLower(name) 167 } 168 return &Bucket{s3, name} 169 } 170 171 var createBucketConfiguration = `<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> 172 <LocationConstraint>%s</LocationConstraint> 173 </CreateBucketConfiguration>` 174 175 // locationConstraint returns an io.Reader specifying a LocationConstraint if 176 // required for the region. 177 // 178 // See http://goo.gl/bh9Kq for details. 179 func (s3 *S3) locationConstraint() io.Reader { 180 constraint := "" 181 if s3.Region.S3LocationConstraint { 182 constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name) 183 } 184 return strings.NewReader(constraint) 185 } 186 187 type ACL string 188 189 const ( 190 Private = ACL("private") 191 PublicRead = ACL("public-read") 192 PublicReadWrite = ACL("public-read-write") 193 AuthenticatedRead = ACL("authenticated-read") 194 BucketOwnerRead = ACL("bucket-owner-read") 195 BucketOwnerFull = ACL("bucket-owner-full-control") 196 ) 197 198 func (b *Bucket) addTokenHeader(headers map[string][]string) { 199 if b.SessionToken == "" { 200 return 201 } 202 headers["x-amz-security-token"] = []string{b.SessionToken} 203 } 204 205 // PutBucket creates a new bucket. 206 // 207 // See http://goo.gl/ndjnR for details. 208 func (b *Bucket) PutBucket(ctx context.Context, perm ACL) error { 209 headers := map[string][]string{ 210 "x-amz-acl": {string(perm)}, 211 } 212 b.addTokenHeader(headers) 213 req := &request{ 214 method: "PUT", 215 bucket: b.Name, 216 path: "/", 217 headers: headers, 218 payload: b.locationConstraint(), 219 } 220 return b.S3.query(ctx, req, nil) 221 } 222 223 // DelBucket removes an existing S3 bucket. All objects in the bucket must 224 // be removed before the bucket itself can be removed. 225 // 226 // See http://goo.gl/GoBrY for details. 227 func (b *Bucket) DelBucket() (err error) { 228 headers := map[string][]string{} 229 b.addTokenHeader(headers) 230 231 req := &request{ 232 method: "DELETE", 233 bucket: b.Name, 234 path: "/", 235 headers: headers, 236 } 237 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 238 err = b.S3.query(context.Background(), req, nil) 239 if !shouldRetry(err) { 240 break 241 } 242 } 243 return err 244 } 245 246 // Get retrieves an object from an S3 bucket. 247 // 248 // See http://goo.gl/isCO7 for details. 249 func (b *Bucket) Get(ctx context.Context, path string) (data []byte, err error) { 250 body, err := b.GetReader(ctx, path) 251 defer func() { 252 if body != nil { 253 body.Close() 254 } 255 }() 256 if err != nil { 257 return nil, err 258 } 259 data, err = io.ReadAll(body) 260 return data, err 261 } 262 263 // GetReader retrieves an object from an S3 bucket, 264 // returning the body of the HTTP response. 265 // It is the caller's responsibility to call Close on rc when 266 // finished reading. 267 func (b *Bucket) GetReader(ctx context.Context, path string) (rc io.ReadCloser, err error) { 268 resp, err := b.GetResponse(ctx, path) 269 if resp != nil { 270 return resp.Body, err 271 } 272 return nil, err 273 } 274 275 // GetReaderWithRange retrieves an object from an S3 bucket using the specified range, 276 // returning the body of the HTTP response. 277 // It is the caller's responsibility to call Close on rc when 278 // finished reading. 279 func (b *Bucket) GetReaderWithRange(ctx context.Context, path string, begin, end int64) (rc io.ReadCloser, err error) { 280 header := make(http.Header) 281 header.Add("Range", fmt.Sprintf("bytes=%d-%d", begin, end-1)) 282 resp, err := b.GetResponseWithHeaders(ctx, path, header) 283 if resp != nil { 284 return resp.Body, err 285 } 286 return nil, err 287 } 288 289 // GetResponse retrieves an object from an S3 bucket, 290 // returning the HTTP response. 291 // It is the caller's responsibility to call Close on rc when 292 // finished reading 293 func (b *Bucket) GetResponse(ctx context.Context, path string) (resp *http.Response, err error) { 294 return b.GetResponseWithHeaders(ctx, path, make(http.Header)) 295 } 296 297 // GetReaderWithHeaders retrieves an object from an S3 bucket 298 // Accepts custom headers to be sent as the second parameter 299 // returning the body of the HTTP response. 300 // It is the caller's responsibility to call Close on rc when 301 // finished reading 302 func (b *Bucket) GetResponseWithHeaders(ctx context.Context, path string, headers map[string][]string) (resp *http.Response, err error) { 303 b.addTokenHeader(headers) 304 req := &request{ 305 bucket: b.Name, 306 path: path, 307 headers: headers, 308 } 309 err = b.S3.prepare(req) 310 if err != nil { 311 return nil, err 312 } 313 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 314 resp, err := b.S3.run(ctx, req, nil) 315 if shouldRetry(err) && attempt.HasNext() { 316 continue 317 } 318 if err != nil { 319 return nil, err 320 } 321 return resp, nil 322 } 323 panic("unreachable") 324 } 325 326 // Exists checks whether or not an object exists on an S3 bucket using a HEAD request. 327 func (b *Bucket) Exists(path string) (exists bool, err error) { 328 headers := map[string][]string{} 329 b.addTokenHeader(headers) 330 331 req := &request{ 332 method: "HEAD", 333 bucket: b.Name, 334 path: path, 335 headers: headers, 336 } 337 err = b.S3.prepare(req) 338 if err != nil { 339 return 340 } 341 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 342 resp, err := b.S3.run(context.Background(), req, nil) 343 344 if shouldRetry(err) && attempt.HasNext() { 345 continue 346 } 347 348 if err != nil { 349 // We can treat a 403 or 404 as non existence 350 if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) { 351 return false, nil 352 } 353 return false, err 354 } 355 356 if resp.StatusCode/100 == 2 { 357 exists = true 358 } 359 return exists, err 360 } 361 return false, fmt.Errorf("S3 Currently Unreachable") 362 } 363 364 // Head HEADs an object in the S3 bucket, returns the response with 365 // no body see http://bit.ly/17K1ylI 366 func (b *Bucket) Head(path string, headers map[string][]string) (*http.Response, error) { 367 b.addTokenHeader(headers) 368 req := &request{ 369 method: "HEAD", 370 bucket: b.Name, 371 path: path, 372 headers: headers, 373 } 374 err := b.S3.prepare(req) 375 if err != nil { 376 return nil, err 377 } 378 379 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 380 resp, err := b.S3.run(context.Background(), req, nil) 381 if shouldRetry(err) && attempt.HasNext() { 382 continue 383 } 384 if err != nil { 385 return nil, err 386 } 387 return resp, err 388 } 389 return nil, fmt.Errorf("S3 Currently Unreachable") 390 } 391 392 // Put inserts an object into the S3 bucket. 393 // 394 // See http://goo.gl/FEBPD for details. 395 func (b *Bucket) Put(ctx context.Context, path string, data []byte, contType string, perm ACL, options Options) error { 396 body := bytes.NewBuffer(data) 397 return b.PutReader(ctx, path, body, int64(len(data)), contType, perm, options) 398 } 399 400 // PutCopy puts a copy of an object given by the key path into bucket b using b.Path as the target key 401 func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (result *CopyObjectResult, err error) { 402 headers := map[string][]string{ 403 "x-amz-acl": {string(perm)}, 404 "x-amz-copy-source": {source}, 405 } 406 b.addTokenHeader(headers) 407 options.addHeaders(headers) 408 req := &request{ 409 method: "PUT", 410 bucket: b.Name, 411 path: path, 412 headers: headers, 413 } 414 result = &CopyObjectResult{} 415 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 416 err = b.S3.query(context.Background(), req, result) 417 if !shouldRetry(err) { 418 break 419 } 420 } 421 if err != nil { 422 return nil, err 423 } 424 return result, nil 425 } 426 427 /* 428 PutHeader - like Put, inserts an object into the S3 bucket. 429 Instead of Content-Type string, pass in custom headers to override defaults. 430 */ 431 func (b *Bucket) PutHeader(ctx context.Context, path string, data []byte, customHeaders map[string][]string, perm ACL) error { 432 body := bytes.NewBuffer(data) 433 return b.PutReaderHeader(ctx, path, body, int64(len(data)), customHeaders, perm) 434 } 435 436 // PutReader inserts an object into the S3 bucket by consuming data 437 // from r until EOF. 438 func (b *Bucket) PutReader(ctx context.Context, path string, r io.Reader, length int64, contType string, perm ACL, options Options) error { 439 headers := map[string][]string{ 440 "Content-Length": {strconv.FormatInt(length, 10)}, 441 "Content-Type": {contType}, 442 "x-amz-acl": {string(perm)}, 443 } 444 b.addTokenHeader(headers) 445 446 options.addHeaders(headers) 447 req := &request{ 448 method: "PUT", 449 bucket: b.Name, 450 path: path, 451 headers: headers, 452 payload: r, 453 } 454 return b.S3.query(ctx, req, nil) 455 } 456 457 /* 458 PutReaderHeader - like PutReader, inserts an object into S3 from a reader. 459 Instead of Content-Type string, pass in custom headers to override defaults. 460 */ 461 func (b *Bucket) PutReaderHeader(ctx context.Context, path string, r io.Reader, length int64, customHeaders map[string][]string, perm ACL) error { 462 // Default headers 463 headers := map[string][]string{ 464 "Content-Length": {strconv.FormatInt(length, 10)}, 465 "Content-Type": {"application/text"}, 466 "x-amz-acl": {string(perm)}, 467 } 468 b.addTokenHeader(headers) 469 470 // Override with custom headers 471 for key, value := range customHeaders { 472 headers[key] = value 473 } 474 475 req := &request{ 476 method: "PUT", 477 bucket: b.Name, 478 path: path, 479 headers: headers, 480 payload: r, 481 } 482 return b.S3.query(ctx, req, nil) 483 } 484 485 // addHeaders adds o's specified fields to headers 486 func (o Options) addHeaders(headers map[string][]string) { 487 if o.SSE { 488 headers["x-amz-server-side-encryption"] = []string{"AES256"} 489 } 490 if len(o.ContentEncoding) != 0 { 491 headers["Content-Encoding"] = []string{o.ContentEncoding} 492 } 493 if len(o.CacheControl) != 0 { 494 headers["Cache-Control"] = []string{o.CacheControl} 495 } 496 if len(o.ContentMD5) != 0 { 497 headers["Content-MD5"] = []string{o.ContentMD5} 498 } 499 if len(o.RedirectLocation) != 0 { 500 headers["x-amz-website-redirect-location"] = []string{o.RedirectLocation} 501 } 502 for k, v := range o.Meta { 503 headers["x-amz-meta-"+k] = v 504 } 505 } 506 507 // addHeaders adds o's specified fields to headers 508 func (o CopyOptions) addHeaders(headers map[string][]string) { 509 o.Options.addHeaders(headers) 510 if len(o.MetadataDirective) != 0 { 511 headers["x-amz-metadata-directive"] = []string{o.MetadataDirective} 512 } 513 if len(o.ContentType) != 0 { 514 headers["Content-Type"] = []string{o.ContentType} 515 } 516 } 517 518 func makeXMLBuffer(doc []byte) *bytes.Buffer { 519 buf := new(bytes.Buffer) 520 buf.WriteString(xml.Header) 521 buf.Write(doc) 522 return buf 523 } 524 525 type RoutingRule struct { 526 ConditionKeyPrefixEquals string `xml:"Condition>KeyPrefixEquals"` 527 RedirectReplaceKeyPrefixWith string `xml:"Redirect>ReplaceKeyPrefixWith,omitempty"` 528 RedirectReplaceKeyWith string `xml:"Redirect>ReplaceKeyWith,omitempty"` 529 } 530 531 type WebsiteConfiguration struct { 532 XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration"` 533 IndexDocumentSuffix string `xml:"IndexDocument>Suffix"` 534 ErrorDocumentKey string `xml:"ErrorDocument>Key"` 535 RoutingRules *[]RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` 536 } 537 538 func (b *Bucket) PutBucketWebsite(configuration WebsiteConfiguration) error { 539 540 doc, err := xml.Marshal(configuration) 541 if err != nil { 542 return err 543 } 544 545 buf := makeXMLBuffer(doc) 546 547 return b.PutBucketSubresource("website", buf, int64(buf.Len())) 548 } 549 550 func (b *Bucket) PutBucketSubresource(subresource string, r io.Reader, length int64) error { 551 headers := map[string][]string{ 552 "Content-Length": {strconv.FormatInt(length, 10)}, 553 } 554 b.addTokenHeader(headers) 555 req := &request{ 556 path: "/", 557 method: "PUT", 558 bucket: b.Name, 559 headers: headers, 560 payload: r, 561 params: url.Values{subresource: {""}}, 562 } 563 564 return b.S3.query(context.Background(), req, nil) 565 } 566 567 // Del removes an object from the S3 bucket. 568 // 569 // See http://goo.gl/APeTt for details. 570 func (b *Bucket) Del(ctx context.Context, path string) error { 571 headers := map[string][]string{} 572 b.addTokenHeader(headers) 573 574 req := &request{ 575 method: "DELETE", 576 bucket: b.Name, 577 path: path, 578 } 579 return b.S3.query(ctx, req, nil) 580 } 581 582 type Delete struct { 583 Quiet bool `xml:"Quiet,omitempty"` 584 Objects []Object `xml:"Object"` 585 } 586 587 type Object struct { 588 Key string `xml:"Key"` 589 VersionID string `xml:"VersionId,omitempty"` 590 } 591 592 // DelMulti removes up to 1000 objects from the S3 bucket. 593 // 594 // See http://goo.gl/jx6cWK for details. 595 func (b *Bucket) DelMulti(objects Delete) error { 596 doc, err := xml.Marshal(objects) 597 if err != nil { 598 return err 599 } 600 601 buf := makeXMLBuffer(doc) 602 digest := md5.New() 603 size, err := digest.Write(buf.Bytes()) 604 if err != nil { 605 return err 606 } 607 608 headers := map[string][]string{ 609 "Content-Length": {strconv.FormatInt(int64(size), 10)}, 610 "Content-MD5": {base64.StdEncoding.EncodeToString(digest.Sum(nil))}, 611 "Content-Type": {"text/xml"}, 612 } 613 b.addTokenHeader(headers) 614 615 req := &request{ 616 path: "/", 617 method: "POST", 618 params: url.Values{"delete": {""}}, 619 bucket: b.Name, 620 headers: headers, 621 payload: buf, 622 } 623 624 return b.S3.query(context.Background(), req, nil) 625 } 626 627 // The ListResp type holds the results of a List bucket operation. 628 type ListResp struct { 629 Name string 630 Prefix string 631 Delimiter string 632 Marker string 633 NextMarker string 634 MaxKeys int 635 636 // IsTruncated is true if the results have been truncated because 637 // there are more keys and prefixes than can fit in MaxKeys. 638 // N.B. this is the opposite sense to that documented (incorrectly) in 639 // http://goo.gl/YjQTc 640 IsTruncated bool 641 Contents []Key 642 CommonPrefixes []string `xml:">Prefix"` 643 } 644 645 // The Key type represents an item stored in an S3 bucket. 646 type Key struct { 647 Key string 648 LastModified string 649 Size int64 650 // ETag gives the hex-encoded MD5 sum of the contents, 651 // surrounded with double-quotes. 652 ETag string 653 StorageClass string 654 Owner Owner 655 } 656 657 // List returns information about objects in an S3 bucket. 658 // 659 // The prefix parameter limits the response to keys that begin with the 660 // specified prefix. 661 // 662 // The delim parameter causes the response to group all of the keys that 663 // share a common prefix up to the next delimiter in a single entry within 664 // the CommonPrefixes field. You can use delimiters to separate a bucket 665 // into different groupings of keys, similar to how folders would work. 666 // 667 // The marker parameter specifies the key to start with when listing objects 668 // in a bucket. Amazon S3 lists objects in alphabetical order and 669 // will return keys alphabetically greater than the marker. 670 // 671 // The max parameter specifies how many keys + common prefixes to return in 672 // the response. The default is 1000. 673 // 674 // For example, given these keys in a bucket: 675 // 676 // index.html 677 // index2.html 678 // photos/2006/January/sample.jpg 679 // photos/2006/February/sample2.jpg 680 // photos/2006/February/sample3.jpg 681 // photos/2006/February/sample4.jpg 682 // 683 // Listing this bucket with delimiter set to "/" would yield the 684 // following result: 685 // 686 // &ListResp{ 687 // Name: "sample-bucket", 688 // MaxKeys: 1000, 689 // Delimiter: "/", 690 // Contents: []Key{ 691 // {Key: "index.html", "index2.html"}, 692 // }, 693 // CommonPrefixes: []string{ 694 // "photos/", 695 // }, 696 // } 697 // 698 // Listing the same bucket with delimiter set to "/" and prefix set to 699 // "photos/2006/" would yield the following result: 700 // 701 // &ListResp{ 702 // Name: "sample-bucket", 703 // MaxKeys: 1000, 704 // Delimiter: "/", 705 // Prefix: "photos/2006/", 706 // CommonPrefixes: []string{ 707 // "photos/2006/February/", 708 // "photos/2006/January/", 709 // }, 710 // } 711 // 712 // See http://goo.gl/YjQTc for details. 713 func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) { 714 params := map[string][]string{ 715 "prefix": {prefix}, 716 "delimiter": {delim}, 717 "marker": {marker}, 718 } 719 if max != 0 { 720 params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} 721 } 722 headers := map[string][]string{} 723 b.addTokenHeader(headers) 724 725 req := &request{ 726 bucket: b.Name, 727 params: params, 728 headers: headers, 729 } 730 result = &ListResp{} 731 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 732 err = b.S3.query(context.Background(), req, result) 733 if !shouldRetry(err) { 734 break 735 } 736 } 737 if err != nil { 738 return nil, err 739 } 740 return result, nil 741 } 742 743 // The VersionsResp type holds the results of a list bucket Versions operation. 744 type VersionsResp struct { 745 Name string 746 Prefix string 747 KeyMarker string 748 VersionIDMarker string `xml:"VersionIdMarker"` 749 MaxKeys int 750 Delimiter string 751 IsTruncated bool 752 Versions []Version 753 CommonPrefixes []string `xml:">Prefix"` 754 } 755 756 // The Version type represents an object version stored in an S3 bucket. 757 type Version struct { 758 Key string 759 VersionID string `xml:"VersionId"` 760 IsLatest bool 761 LastModified string 762 // ETag gives the hex-encoded MD5 sum of the contents, 763 // surrounded with double-quotes. 764 ETag string 765 Size int64 766 Owner Owner 767 StorageClass string 768 } 769 770 func (b *Bucket) Versions(prefix, delim, keyMarker string, versionIDMarker string, max int) (result *VersionsResp, err error) { 771 params := map[string][]string{ 772 "versions": {""}, 773 "prefix": {prefix}, 774 "delimiter": {delim}, 775 } 776 777 if len(versionIDMarker) != 0 { 778 params["version-id-marker"] = []string{versionIDMarker} 779 } 780 if len(keyMarker) != 0 { 781 params["key-marker"] = []string{keyMarker} 782 } 783 784 if max != 0 { 785 params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} 786 } 787 req := &request{ 788 bucket: b.Name, 789 params: params, 790 } 791 result = &VersionsResp{} 792 for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { 793 err = b.S3.query(context.Background(), req, result) 794 if !shouldRetry(err) { 795 break 796 } 797 } 798 if err != nil { 799 return nil, err 800 } 801 return result, nil 802 } 803 804 // Returns a mapping of all key names in this bucket to Key objects 805 func (b *Bucket) GetBucketContents() (*map[string]Key, error) { 806 bucketContents := map[string]Key{} 807 prefix := "" 808 pathSeparator := "" 809 marker := "" 810 for { 811 contents, err := b.List(prefix, pathSeparator, marker, 1000) 812 if err != nil { 813 return &bucketContents, err 814 } 815 for _, key := range contents.Contents { 816 bucketContents[key.Key] = key 817 } 818 if contents.IsTruncated { 819 marker = contents.NextMarker 820 } else { 821 break 822 } 823 } 824 825 return &bucketContents, nil 826 } 827 828 // URL returns a non-signed URL that allows retrieving the 829 // object at path. It only works if the object is publicly 830 // readable (see SignedURL). 831 func (b *Bucket) URL(path string) string { 832 req := &request{ 833 bucket: b.Name, 834 path: path, 835 } 836 err := b.S3.prepare(req) 837 if err != nil { 838 panic(err) 839 } 840 u, err := req.url() 841 if err != nil { 842 panic(err) 843 } 844 u.RawQuery = "" 845 return u.String() 846 } 847 848 // SignedURL returns a signed URL that allows anyone holding the URL 849 // to retrieve the object at path. The signature is valid until expires. 850 func (b *Bucket) SignedURL(path string, expires time.Time) string { 851 req := &request{ 852 bucket: b.Name, 853 path: path, 854 params: url.Values{"Expires": {strconv.FormatInt(expires.Unix(), 10)}}, 855 } 856 err := b.S3.prepare(req) 857 if err != nil { 858 panic(err) 859 } 860 u, err := req.url() 861 if err != nil { 862 panic(err) 863 } 864 return u.String() 865 } 866 867 type request struct { 868 method string 869 bucket string 870 path string 871 params url.Values 872 headers http.Header 873 baseurl string 874 payload io.Reader 875 prepared bool 876 } 877 878 func (req *request) url() (*url.URL, error) { 879 u, err := url.Parse(req.baseurl) 880 if err != nil { 881 return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) 882 } 883 u.RawQuery = req.params.Encode() 884 u.Path = req.path 885 return u, nil 886 } 887 888 // query prepares and runs the req request. 889 // If resp is not nil, the XML data contained in the response 890 // body will be unmarshalled on it. 891 func (s3 *S3) query(ctx context.Context, req *request, resp interface{}) error { 892 err := s3.prepare(req) 893 if err == nil { 894 var httpResponse *http.Response 895 httpResponse, err = s3.run(ctx, req, resp) 896 if resp == nil && httpResponse != nil { 897 httpResponse.Body.Close() 898 } 899 } 900 return err 901 } 902 903 // prepare sets up req to be delivered to S3. 904 func (s3 *S3) prepare(req *request) error { 905 var signpath = req.path 906 907 if !req.prepared { 908 req.prepared = true 909 if req.method == "" { 910 req.method = "GET" 911 } 912 // Copy so they can be mutated without affecting on retries. 913 params := make(url.Values) 914 headers := make(http.Header) 915 for k, v := range req.params { 916 params[k] = v 917 } 918 for k, v := range req.headers { 919 headers[k] = v 920 } 921 req.params = params 922 req.headers = headers 923 if !strings.HasPrefix(req.path, "/") { 924 req.path = "/" + req.path 925 } 926 signpath = req.path 927 if req.bucket != "" { 928 req.baseurl = s3.Region.S3BucketEndpoint 929 if req.baseurl == "" { 930 // Use the path method to address the bucket. 931 req.baseurl = s3.Region.S3Endpoint 932 req.path = "/" + req.bucket + req.path 933 } else { 934 // Just in case, prevent injection. 935 if strings.ContainsAny(req.bucket, "/:@") { 936 return fmt.Errorf("bad S3 bucket: %q", req.bucket) 937 } 938 req.baseurl = strings.ReplaceAll(req.baseurl, "${bucket}", req.bucket) 939 } 940 signpath = "/" + req.bucket + signpath 941 } 942 } 943 944 // Always sign again as it's not clear how far the 945 // server has handled a previous attempt. 946 u, err := url.Parse(req.baseurl) 947 if err != nil { 948 return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) 949 } 950 reqSignpathSpaceFix := (&url.URL{Path: signpath}).String() 951 req.headers["Host"] = []string{u.Host} 952 req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} 953 return s3.sign(req.method, reqSignpathSpaceFix, req.params, req.headers) 954 } 955 956 // run sends req and returns the http response from the server. 957 // If resp is not nil, the XML data contained in the response 958 // body will be unmarshalled on it. 959 func (s3 *S3) run(ctx context.Context, req *request, resp interface{}) (*http.Response, error) { 960 if debug { 961 log.Printf("Running S3 request: %#v", req) 962 } 963 964 u, err := req.url() 965 if err != nil { 966 return nil, err 967 } 968 969 hreq := &http.Request{ 970 URL: u, 971 Method: req.method, 972 ProtoMajor: 1, 973 ProtoMinor: 1, 974 Close: true, 975 Header: req.headers, 976 } 977 978 if ctx != nil { 979 hreq = hreq.WithContext(ctx) 980 } 981 982 if v, ok := req.headers["Content-Length"]; ok { 983 hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64) 984 delete(req.headers, "Content-Length") 985 } 986 if req.payload != nil { 987 hreq.Body = io.NopCloser(req.payload) 988 } 989 990 if s3.client == nil { 991 s3.client = &http.Client{ 992 Transport: &http.Transport{ 993 Dial: func(netw, addr string) (c net.Conn, err error) { 994 c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout) 995 if err != nil { 996 return 997 } 998 999 var deadline time.Time 1000 if s3.RequestTimeout > 0 { 1001 deadline = time.Now().Add(s3.RequestTimeout) 1002 err := c.SetDeadline(deadline) 1003 if err != nil { 1004 return nil, err 1005 } 1006 } 1007 1008 if s3.ReadTimeout > 0 || s3.WriteTimeout > 0 { 1009 c = &ioTimeoutConn{ 1010 TCPConn: c.(*net.TCPConn), 1011 readTimeout: s3.ReadTimeout, 1012 writeTimeout: s3.WriteTimeout, 1013 requestDeadline: deadline, 1014 } 1015 } 1016 return 1017 }, 1018 Proxy: libkb.MakeProxy(s3.G().Env), 1019 }, 1020 } 1021 } 1022 1023 hresp, err := s3.client.Do(hreq) 1024 if err != nil { 1025 return nil, err 1026 } 1027 if debug { 1028 dump, _ := httputil.DumpResponse(hresp, true) 1029 log.Printf("} -> %s\n", dump) 1030 } 1031 if hresp.StatusCode != 200 && hresp.StatusCode != 204 && hresp.StatusCode != 206 { 1032 defer hresp.Body.Close() 1033 return nil, buildError(hresp) 1034 } 1035 if resp != nil { 1036 err = xml.NewDecoder(hresp.Body).Decode(resp) 1037 hresp.Body.Close() 1038 if debug { 1039 log.Printf("goamz.s3> decoded xml into %#v", resp) 1040 } 1041 } 1042 return hresp, err 1043 } 1044 1045 // Error represents an error in an operation with S3. 1046 type Error struct { 1047 StatusCode int // HTTP status code (200, 403, ...) 1048 Code string // EC2 error code ("UnsupportedOperation", ...) 1049 Message string // The human-oriented error message 1050 BucketName string 1051 RequestID string `xml:"RequestId"` 1052 HostID string `xml:"HostId"` 1053 } 1054 1055 func (e *Error) Error() string { 1056 return e.Message 1057 } 1058 1059 func buildError(r *http.Response) error { 1060 if debug { 1061 log.Printf("got error (status code %v)", r.StatusCode) 1062 data, err := io.ReadAll(r.Body) 1063 if err != nil { 1064 log.Printf("\tread error: %v", err) 1065 } else { 1066 log.Printf("\tdata:\n%s\n\n", data) 1067 } 1068 r.Body = io.NopCloser(bytes.NewBuffer(data)) 1069 } 1070 1071 err := Error{} 1072 1073 // TODO return error if Unmarshal fails? 1074 decodeErr := xml.NewDecoder(r.Body).Decode(&err) 1075 if decodeErr != nil { 1076 log.Printf("\tdecodeErr error: %v", decodeErr) 1077 } 1078 r.Body.Close() 1079 err.StatusCode = r.StatusCode 1080 if err.Message == "" { 1081 err.Message = r.Status 1082 } 1083 if debug { 1084 log.Printf("err: %#v\n", err) 1085 } 1086 return &err 1087 } 1088 1089 func shouldRetry(err error) bool { 1090 if err == nil { 1091 return false 1092 } 1093 if e, ok := err.(*url.Error); ok { 1094 // Transport returns this string if it detects a write on a connection which 1095 // has already had an error 1096 if e.Err.Error() == "http: can't write HTTP request on broken connection" { 1097 return true 1098 } 1099 err = e.Err 1100 } 1101 1102 switch err { 1103 case io.ErrUnexpectedEOF, io.EOF: 1104 return true 1105 } 1106 switch e := err.(type) { 1107 case *net.DNSError: 1108 return true 1109 case *net.OpError: 1110 switch e.Op { 1111 case "read", "write", "WSARecv", "WSASend", "ConnectEx": 1112 return true 1113 } 1114 case *Error: 1115 switch e.Code { 1116 case "InternalError", "NoSuchUpload", "NoSuchBucket", "RequestTimeout": 1117 return true 1118 } 1119 // let's handle tls handshake timeout issues and similar temporary errors 1120 case net.Error: 1121 return e.Temporary() //nolint 1122 } 1123 1124 return false 1125 } 1126 1127 func hasCode(err error, code string) bool { 1128 s3err, ok := err.(*Error) 1129 return ok && s3err.Code == code 1130 } 1131 1132 // ioTimeoutConn is a net.Conn which sets a deadline for each Read or Write operation 1133 type ioTimeoutConn struct { 1134 *net.TCPConn 1135 readTimeout time.Duration 1136 writeTimeout time.Duration 1137 requestDeadline time.Time 1138 } 1139 1140 func (c *ioTimeoutConn) deadline(timeout time.Duration) time.Time { 1141 dl := time.Now().Add(timeout) 1142 if c.requestDeadline.IsZero() || dl.Before(c.requestDeadline) { 1143 return dl 1144 } 1145 1146 return c.requestDeadline 1147 } 1148 1149 func (c *ioTimeoutConn) Read(b []byte) (int, error) { 1150 if c.readTimeout > 0 { 1151 err := c.TCPConn.SetReadDeadline(c.deadline(c.readTimeout)) 1152 if err != nil { 1153 return 0, err 1154 } 1155 } 1156 return c.TCPConn.Read(b) 1157 } 1158 1159 func (c *ioTimeoutConn) Write(b []byte) (int, error) { 1160 if c.writeTimeout > 0 { 1161 err := c.TCPConn.SetWriteDeadline(c.deadline(c.writeTimeout)) 1162 if err != nil { 1163 return 0, err 1164 } 1165 } 1166 return c.TCPConn.Write(b) 1167 }