github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/chat/s3/s3.go (about)

     1  //
     2  // goamz - Go packages to interact with the Amazon Web Services.
     3  //
     4  //   https://wiki.ubuntu.com/goamz
     5  //
     6  // Copyright (c) 2011 Canonical Ltd.
     7  //
     8  // Written by Gustavo Niemeyer <gustavo.niemeyer@canonical.com>
     9  //
    10  // Modified by Keybase to allow external signing of requests.
    11  
    12  package s3
    13  
    14  import (
    15  	"bytes"
    16  	"crypto/md5"
    17  	"encoding/base64"
    18  	"encoding/xml"
    19  	"fmt"
    20  	"io"
    21  	"log"
    22  	"net"
    23  	"net/http"
    24  	"net/http/httputil"
    25  	"net/url"
    26  	"strconv"
    27  	"strings"
    28  	"time"
    29  
    30  	"github.com/keybase/client/go/libkb"
    31  
    32  	"golang.org/x/net/context"
    33  )
    34  
    35  const debug = false
    36  
    37  type Signer interface {
    38  	Sign(payload []byte) ([]byte, error)
    39  }
    40  
    41  // The S3 type encapsulates operations with an S3 region.
    42  type S3 struct {
    43  	Region
    44  	libkb.Contextified
    45  
    46  	// This is needed for payload construction.  It's
    47  	// ok for clients to know it.
    48  	AccessKey string
    49  
    50  	// This is needed for payload construction.  It's
    51  	// ok for clients to know it.
    52  	SessionToken string
    53  
    54  	// Signer signs payloads for s3 request authorization.
    55  	Signer Signer
    56  
    57  	// ConnectTimeout is the maximum time a request attempt will
    58  	// wait for a successful connection to be made.
    59  	//
    60  	// A value of zero means no timeout.
    61  	ConnectTimeout time.Duration
    62  
    63  	// ReadTimeout is the maximum time a request attempt will wait
    64  	// for an individual read to complete.
    65  	//
    66  	// A value of zero means no timeout.
    67  	ReadTimeout time.Duration
    68  
    69  	// WriteTimeout is the maximum time a request attempt will
    70  	// wait for an individual write to complete.
    71  	//
    72  	// A value of zero means no timeout.
    73  	WriteTimeout time.Duration
    74  
    75  	// RequestTimeout is the maximum time a request attempt can
    76  	// take before operations return a timeout error.
    77  	//
    78  	// This includes connection time, any redirects, and reading
    79  	// the response body. The timer remains running after the request
    80  	// is made so it can interrupt reading of the response data.
    81  	//
    82  	// A Timeout of zero means no timeout.
    83  	RequestTimeout time.Duration
    84  
    85  	// AttemptStrategy is the attempt strategy used for requests.
    86  	AttemptStrategy
    87  
    88  	// client used for requests
    89  	client *http.Client
    90  }
    91  
    92  // The Bucket type encapsulates operations with an S3 bucket.
    93  type Bucket struct {
    94  	*S3
    95  	Name string
    96  }
    97  
    98  // The Owner type represents the owner of the object in an S3 bucket.
    99  type Owner struct {
   100  	ID          string
   101  	DisplayName string
   102  }
   103  
   104  // Fold options into an Options struct
   105  type Options struct {
   106  	SSE              bool
   107  	Meta             map[string][]string
   108  	ContentEncoding  string
   109  	CacheControl     string
   110  	RedirectLocation string
   111  	ContentMD5       string
   112  	// What else?
   113  	// Content-Disposition string
   114  	//// The following become headers so they are []strings rather than strings... I think
   115  	// x-amz-storage-class []string
   116  }
   117  
   118  type CopyOptions struct {
   119  	Options
   120  	MetadataDirective string
   121  	ContentType       string
   122  }
   123  
   124  // CopyObjectResult is the output from a Copy request
   125  type CopyObjectResult struct {
   126  	ETag         string
   127  	LastModified string
   128  }
   129  
   130  // DefaultAttemptStrategy is the default AttemptStrategy used by S3 objects created by New.
   131  var DefaultAttemptStrategy = AttemptStrategy{
   132  	Min:   5,
   133  	Total: 5 * time.Second,
   134  	Delay: 200 * time.Millisecond,
   135  }
   136  
   137  // New creates a new S3.  Optional client argument allows for custom http.clients to be used.
   138  func New(g *libkb.GlobalContext, signer Signer, region Region, client ...*http.Client) *S3 {
   139  
   140  	var httpclient *http.Client
   141  
   142  	if len(client) > 0 {
   143  		httpclient = client[0]
   144  	}
   145  
   146  	return &S3{
   147  		Signer:          signer,
   148  		Region:          region,
   149  		AttemptStrategy: DefaultAttemptStrategy,
   150  		client:          httpclient,
   151  		Contextified:    libkb.NewContextified(g),
   152  	}
   153  }
   154  
   155  func (s3 *S3) SetAccessKey(key string) {
   156  	s3.AccessKey = key
   157  }
   158  
   159  func (s3 *S3) SetSessionToken(token string) {
   160  	s3.SessionToken = token
   161  }
   162  
   163  // Bucket returns a Bucket with the given name.
   164  func (s3 *S3) Bucket(name string) BucketInt {
   165  	if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket {
   166  		name = strings.ToLower(name)
   167  	}
   168  	return &Bucket{s3, name}
   169  }
   170  
   171  var createBucketConfiguration = `<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
   172    <LocationConstraint>%s</LocationConstraint>
   173  </CreateBucketConfiguration>`
   174  
   175  // locationConstraint returns an io.Reader specifying a LocationConstraint if
   176  // required for the region.
   177  //
   178  // See http://goo.gl/bh9Kq for details.
   179  func (s3 *S3) locationConstraint() io.Reader {
   180  	constraint := ""
   181  	if s3.Region.S3LocationConstraint {
   182  		constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name)
   183  	}
   184  	return strings.NewReader(constraint)
   185  }
   186  
   187  type ACL string
   188  
   189  const (
   190  	Private           = ACL("private")
   191  	PublicRead        = ACL("public-read")
   192  	PublicReadWrite   = ACL("public-read-write")
   193  	AuthenticatedRead = ACL("authenticated-read")
   194  	BucketOwnerRead   = ACL("bucket-owner-read")
   195  	BucketOwnerFull   = ACL("bucket-owner-full-control")
   196  )
   197  
   198  func (b *Bucket) addTokenHeader(headers map[string][]string) {
   199  	if b.SessionToken == "" {
   200  		return
   201  	}
   202  	headers["x-amz-security-token"] = []string{b.SessionToken}
   203  }
   204  
   205  // PutBucket creates a new bucket.
   206  //
   207  // See http://goo.gl/ndjnR for details.
   208  func (b *Bucket) PutBucket(ctx context.Context, perm ACL) error {
   209  	headers := map[string][]string{
   210  		"x-amz-acl": {string(perm)},
   211  	}
   212  	b.addTokenHeader(headers)
   213  	req := &request{
   214  		method:  "PUT",
   215  		bucket:  b.Name,
   216  		path:    "/",
   217  		headers: headers,
   218  		payload: b.locationConstraint(),
   219  	}
   220  	return b.S3.query(ctx, req, nil)
   221  }
   222  
   223  // DelBucket removes an existing S3 bucket. All objects in the bucket must
   224  // be removed before the bucket itself can be removed.
   225  //
   226  // See http://goo.gl/GoBrY for details.
   227  func (b *Bucket) DelBucket() (err error) {
   228  	headers := map[string][]string{}
   229  	b.addTokenHeader(headers)
   230  
   231  	req := &request{
   232  		method:  "DELETE",
   233  		bucket:  b.Name,
   234  		path:    "/",
   235  		headers: headers,
   236  	}
   237  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   238  		err = b.S3.query(context.Background(), req, nil)
   239  		if !shouldRetry(err) {
   240  			break
   241  		}
   242  	}
   243  	return err
   244  }
   245  
   246  // Get retrieves an object from an S3 bucket.
   247  //
   248  // See http://goo.gl/isCO7 for details.
   249  func (b *Bucket) Get(ctx context.Context, path string) (data []byte, err error) {
   250  	body, err := b.GetReader(ctx, path)
   251  	defer func() {
   252  		if body != nil {
   253  			body.Close()
   254  		}
   255  	}()
   256  	if err != nil {
   257  		return nil, err
   258  	}
   259  	data, err = io.ReadAll(body)
   260  	return data, err
   261  }
   262  
   263  // GetReader retrieves an object from an S3 bucket,
   264  // returning the body of the HTTP response.
   265  // It is the caller's responsibility to call Close on rc when
   266  // finished reading.
   267  func (b *Bucket) GetReader(ctx context.Context, path string) (rc io.ReadCloser, err error) {
   268  	resp, err := b.GetResponse(ctx, path)
   269  	if resp != nil {
   270  		return resp.Body, err
   271  	}
   272  	return nil, err
   273  }
   274  
   275  // GetReaderWithRange retrieves an object from an S3 bucket using the specified range,
   276  // returning the body of the HTTP response.
   277  // It is the caller's responsibility to call Close on rc when
   278  // finished reading.
   279  func (b *Bucket) GetReaderWithRange(ctx context.Context, path string, begin, end int64) (rc io.ReadCloser, err error) {
   280  	header := make(http.Header)
   281  	header.Add("Range", fmt.Sprintf("bytes=%d-%d", begin, end-1))
   282  	resp, err := b.GetResponseWithHeaders(ctx, path, header)
   283  	if resp != nil {
   284  		return resp.Body, err
   285  	}
   286  	return nil, err
   287  }
   288  
   289  // GetResponse retrieves an object from an S3 bucket,
   290  // returning the HTTP response.
   291  // It is the caller's responsibility to call Close on rc when
   292  // finished reading
   293  func (b *Bucket) GetResponse(ctx context.Context, path string) (resp *http.Response, err error) {
   294  	return b.GetResponseWithHeaders(ctx, path, make(http.Header))
   295  }
   296  
   297  // GetReaderWithHeaders retrieves an object from an S3 bucket
   298  // Accepts custom headers to be sent as the second parameter
   299  // returning the body of the HTTP response.
   300  // It is the caller's responsibility to call Close on rc when
   301  // finished reading
   302  func (b *Bucket) GetResponseWithHeaders(ctx context.Context, path string, headers map[string][]string) (resp *http.Response, err error) {
   303  	b.addTokenHeader(headers)
   304  	req := &request{
   305  		bucket:  b.Name,
   306  		path:    path,
   307  		headers: headers,
   308  	}
   309  	err = b.S3.prepare(req)
   310  	if err != nil {
   311  		return nil, err
   312  	}
   313  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   314  		resp, err := b.S3.run(ctx, req, nil)
   315  		if shouldRetry(err) && attempt.HasNext() {
   316  			continue
   317  		}
   318  		if err != nil {
   319  			return nil, err
   320  		}
   321  		return resp, nil
   322  	}
   323  	panic("unreachable")
   324  }
   325  
   326  // Exists checks whether or not an object exists on an S3 bucket using a HEAD request.
   327  func (b *Bucket) Exists(path string) (exists bool, err error) {
   328  	headers := map[string][]string{}
   329  	b.addTokenHeader(headers)
   330  
   331  	req := &request{
   332  		method:  "HEAD",
   333  		bucket:  b.Name,
   334  		path:    path,
   335  		headers: headers,
   336  	}
   337  	err = b.S3.prepare(req)
   338  	if err != nil {
   339  		return
   340  	}
   341  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   342  		resp, err := b.S3.run(context.Background(), req, nil)
   343  
   344  		if shouldRetry(err) && attempt.HasNext() {
   345  			continue
   346  		}
   347  
   348  		if err != nil {
   349  			// We can treat a 403 or 404 as non existence
   350  			if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
   351  				return false, nil
   352  			}
   353  			return false, err
   354  		}
   355  
   356  		if resp.StatusCode/100 == 2 {
   357  			exists = true
   358  		}
   359  		return exists, err
   360  	}
   361  	return false, fmt.Errorf("S3 Currently Unreachable")
   362  }
   363  
   364  // Head HEADs an object in the S3 bucket, returns the response with
   365  // no body see http://bit.ly/17K1ylI
   366  func (b *Bucket) Head(path string, headers map[string][]string) (*http.Response, error) {
   367  	b.addTokenHeader(headers)
   368  	req := &request{
   369  		method:  "HEAD",
   370  		bucket:  b.Name,
   371  		path:    path,
   372  		headers: headers,
   373  	}
   374  	err := b.S3.prepare(req)
   375  	if err != nil {
   376  		return nil, err
   377  	}
   378  
   379  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   380  		resp, err := b.S3.run(context.Background(), req, nil)
   381  		if shouldRetry(err) && attempt.HasNext() {
   382  			continue
   383  		}
   384  		if err != nil {
   385  			return nil, err
   386  		}
   387  		return resp, err
   388  	}
   389  	return nil, fmt.Errorf("S3 Currently Unreachable")
   390  }
   391  
   392  // Put inserts an object into the S3 bucket.
   393  //
   394  // See http://goo.gl/FEBPD for details.
   395  func (b *Bucket) Put(ctx context.Context, path string, data []byte, contType string, perm ACL, options Options) error {
   396  	body := bytes.NewBuffer(data)
   397  	return b.PutReader(ctx, path, body, int64(len(data)), contType, perm, options)
   398  }
   399  
   400  // PutCopy puts a copy of an object given by the key path into bucket b using b.Path as the target key
   401  func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (result *CopyObjectResult, err error) {
   402  	headers := map[string][]string{
   403  		"x-amz-acl":         {string(perm)},
   404  		"x-amz-copy-source": {source},
   405  	}
   406  	b.addTokenHeader(headers)
   407  	options.addHeaders(headers)
   408  	req := &request{
   409  		method:  "PUT",
   410  		bucket:  b.Name,
   411  		path:    path,
   412  		headers: headers,
   413  	}
   414  	result = &CopyObjectResult{}
   415  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   416  		err = b.S3.query(context.Background(), req, result)
   417  		if !shouldRetry(err) {
   418  			break
   419  		}
   420  	}
   421  	if err != nil {
   422  		return nil, err
   423  	}
   424  	return result, nil
   425  }
   426  
   427  /*
   428  PutHeader - like Put, inserts an object into the S3 bucket.
   429  Instead of Content-Type string, pass in custom headers to override defaults.
   430  */
   431  func (b *Bucket) PutHeader(ctx context.Context, path string, data []byte, customHeaders map[string][]string, perm ACL) error {
   432  	body := bytes.NewBuffer(data)
   433  	return b.PutReaderHeader(ctx, path, body, int64(len(data)), customHeaders, perm)
   434  }
   435  
   436  // PutReader inserts an object into the S3 bucket by consuming data
   437  // from r until EOF.
   438  func (b *Bucket) PutReader(ctx context.Context, path string, r io.Reader, length int64, contType string, perm ACL, options Options) error {
   439  	headers := map[string][]string{
   440  		"Content-Length": {strconv.FormatInt(length, 10)},
   441  		"Content-Type":   {contType},
   442  		"x-amz-acl":      {string(perm)},
   443  	}
   444  	b.addTokenHeader(headers)
   445  
   446  	options.addHeaders(headers)
   447  	req := &request{
   448  		method:  "PUT",
   449  		bucket:  b.Name,
   450  		path:    path,
   451  		headers: headers,
   452  		payload: r,
   453  	}
   454  	return b.S3.query(ctx, req, nil)
   455  }
   456  
   457  /*
   458  PutReaderHeader - like PutReader, inserts an object into S3 from a reader.
   459  Instead of Content-Type string, pass in custom headers to override defaults.
   460  */
   461  func (b *Bucket) PutReaderHeader(ctx context.Context, path string, r io.Reader, length int64, customHeaders map[string][]string, perm ACL) error {
   462  	// Default headers
   463  	headers := map[string][]string{
   464  		"Content-Length": {strconv.FormatInt(length, 10)},
   465  		"Content-Type":   {"application/text"},
   466  		"x-amz-acl":      {string(perm)},
   467  	}
   468  	b.addTokenHeader(headers)
   469  
   470  	// Override with custom headers
   471  	for key, value := range customHeaders {
   472  		headers[key] = value
   473  	}
   474  
   475  	req := &request{
   476  		method:  "PUT",
   477  		bucket:  b.Name,
   478  		path:    path,
   479  		headers: headers,
   480  		payload: r,
   481  	}
   482  	return b.S3.query(ctx, req, nil)
   483  }
   484  
   485  // addHeaders adds o's specified fields to headers
   486  func (o Options) addHeaders(headers map[string][]string) {
   487  	if o.SSE {
   488  		headers["x-amz-server-side-encryption"] = []string{"AES256"}
   489  	}
   490  	if len(o.ContentEncoding) != 0 {
   491  		headers["Content-Encoding"] = []string{o.ContentEncoding}
   492  	}
   493  	if len(o.CacheControl) != 0 {
   494  		headers["Cache-Control"] = []string{o.CacheControl}
   495  	}
   496  	if len(o.ContentMD5) != 0 {
   497  		headers["Content-MD5"] = []string{o.ContentMD5}
   498  	}
   499  	if len(o.RedirectLocation) != 0 {
   500  		headers["x-amz-website-redirect-location"] = []string{o.RedirectLocation}
   501  	}
   502  	for k, v := range o.Meta {
   503  		headers["x-amz-meta-"+k] = v
   504  	}
   505  }
   506  
   507  // addHeaders adds o's specified fields to headers
   508  func (o CopyOptions) addHeaders(headers map[string][]string) {
   509  	o.Options.addHeaders(headers)
   510  	if len(o.MetadataDirective) != 0 {
   511  		headers["x-amz-metadata-directive"] = []string{o.MetadataDirective}
   512  	}
   513  	if len(o.ContentType) != 0 {
   514  		headers["Content-Type"] = []string{o.ContentType}
   515  	}
   516  }
   517  
   518  func makeXMLBuffer(doc []byte) *bytes.Buffer {
   519  	buf := new(bytes.Buffer)
   520  	buf.WriteString(xml.Header)
   521  	buf.Write(doc)
   522  	return buf
   523  }
   524  
   525  type RoutingRule struct {
   526  	ConditionKeyPrefixEquals     string `xml:"Condition>KeyPrefixEquals"`
   527  	RedirectReplaceKeyPrefixWith string `xml:"Redirect>ReplaceKeyPrefixWith,omitempty"`
   528  	RedirectReplaceKeyWith       string `xml:"Redirect>ReplaceKeyWith,omitempty"`
   529  }
   530  
   531  type WebsiteConfiguration struct {
   532  	XMLName             xml.Name       `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration"`
   533  	IndexDocumentSuffix string         `xml:"IndexDocument>Suffix"`
   534  	ErrorDocumentKey    string         `xml:"ErrorDocument>Key"`
   535  	RoutingRules        *[]RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"`
   536  }
   537  
   538  func (b *Bucket) PutBucketWebsite(configuration WebsiteConfiguration) error {
   539  
   540  	doc, err := xml.Marshal(configuration)
   541  	if err != nil {
   542  		return err
   543  	}
   544  
   545  	buf := makeXMLBuffer(doc)
   546  
   547  	return b.PutBucketSubresource("website", buf, int64(buf.Len()))
   548  }
   549  
   550  func (b *Bucket) PutBucketSubresource(subresource string, r io.Reader, length int64) error {
   551  	headers := map[string][]string{
   552  		"Content-Length": {strconv.FormatInt(length, 10)},
   553  	}
   554  	b.addTokenHeader(headers)
   555  	req := &request{
   556  		path:    "/",
   557  		method:  "PUT",
   558  		bucket:  b.Name,
   559  		headers: headers,
   560  		payload: r,
   561  		params:  url.Values{subresource: {""}},
   562  	}
   563  
   564  	return b.S3.query(context.Background(), req, nil)
   565  }
   566  
   567  // Del removes an object from the S3 bucket.
   568  //
   569  // See http://goo.gl/APeTt for details.
   570  func (b *Bucket) Del(ctx context.Context, path string) error {
   571  	headers := map[string][]string{}
   572  	b.addTokenHeader(headers)
   573  
   574  	req := &request{
   575  		method: "DELETE",
   576  		bucket: b.Name,
   577  		path:   path,
   578  	}
   579  	return b.S3.query(ctx, req, nil)
   580  }
   581  
   582  type Delete struct {
   583  	Quiet   bool     `xml:"Quiet,omitempty"`
   584  	Objects []Object `xml:"Object"`
   585  }
   586  
   587  type Object struct {
   588  	Key       string `xml:"Key"`
   589  	VersionID string `xml:"VersionId,omitempty"`
   590  }
   591  
   592  // DelMulti removes up to 1000 objects from the S3 bucket.
   593  //
   594  // See http://goo.gl/jx6cWK for details.
   595  func (b *Bucket) DelMulti(objects Delete) error {
   596  	doc, err := xml.Marshal(objects)
   597  	if err != nil {
   598  		return err
   599  	}
   600  
   601  	buf := makeXMLBuffer(doc)
   602  	digest := md5.New()
   603  	size, err := digest.Write(buf.Bytes())
   604  	if err != nil {
   605  		return err
   606  	}
   607  
   608  	headers := map[string][]string{
   609  		"Content-Length": {strconv.FormatInt(int64(size), 10)},
   610  		"Content-MD5":    {base64.StdEncoding.EncodeToString(digest.Sum(nil))},
   611  		"Content-Type":   {"text/xml"},
   612  	}
   613  	b.addTokenHeader(headers)
   614  
   615  	req := &request{
   616  		path:    "/",
   617  		method:  "POST",
   618  		params:  url.Values{"delete": {""}},
   619  		bucket:  b.Name,
   620  		headers: headers,
   621  		payload: buf,
   622  	}
   623  
   624  	return b.S3.query(context.Background(), req, nil)
   625  }
   626  
   627  // The ListResp type holds the results of a List bucket operation.
   628  type ListResp struct {
   629  	Name       string
   630  	Prefix     string
   631  	Delimiter  string
   632  	Marker     string
   633  	NextMarker string
   634  	MaxKeys    int
   635  
   636  	// IsTruncated is true if the results have been truncated because
   637  	// there are more keys and prefixes than can fit in MaxKeys.
   638  	// N.B. this is the opposite sense to that documented (incorrectly) in
   639  	// http://goo.gl/YjQTc
   640  	IsTruncated    bool
   641  	Contents       []Key
   642  	CommonPrefixes []string `xml:">Prefix"`
   643  }
   644  
   645  // The Key type represents an item stored in an S3 bucket.
   646  type Key struct {
   647  	Key          string
   648  	LastModified string
   649  	Size         int64
   650  	// ETag gives the hex-encoded MD5 sum of the contents,
   651  	// surrounded with double-quotes.
   652  	ETag         string
   653  	StorageClass string
   654  	Owner        Owner
   655  }
   656  
   657  // List returns information about objects in an S3 bucket.
   658  //
   659  // The prefix parameter limits the response to keys that begin with the
   660  // specified prefix.
   661  //
   662  // The delim parameter causes the response to group all of the keys that
   663  // share a common prefix up to the next delimiter in a single entry within
   664  // the CommonPrefixes field. You can use delimiters to separate a bucket
   665  // into different groupings of keys, similar to how folders would work.
   666  //
   667  // The marker parameter specifies the key to start with when listing objects
   668  // in a bucket. Amazon S3 lists objects in alphabetical order and
   669  // will return keys alphabetically greater than the marker.
   670  //
   671  // The max parameter specifies how many keys + common prefixes to return in
   672  // the response. The default is 1000.
   673  //
   674  // For example, given these keys in a bucket:
   675  //
   676  //	index.html
   677  //	index2.html
   678  //	photos/2006/January/sample.jpg
   679  //	photos/2006/February/sample2.jpg
   680  //	photos/2006/February/sample3.jpg
   681  //	photos/2006/February/sample4.jpg
   682  //
   683  // Listing this bucket with delimiter set to "/" would yield the
   684  // following result:
   685  //
   686  //	&ListResp{
   687  //	    Name:      "sample-bucket",
   688  //	    MaxKeys:   1000,
   689  //	    Delimiter: "/",
   690  //	    Contents:  []Key{
   691  //	        {Key: "index.html", "index2.html"},
   692  //	    },
   693  //	    CommonPrefixes: []string{
   694  //	        "photos/",
   695  //	    },
   696  //	}
   697  //
   698  // Listing the same bucket with delimiter set to "/" and prefix set to
   699  // "photos/2006/" would yield the following result:
   700  //
   701  //	&ListResp{
   702  //	    Name:      "sample-bucket",
   703  //	    MaxKeys:   1000,
   704  //	    Delimiter: "/",
   705  //	    Prefix:    "photos/2006/",
   706  //	    CommonPrefixes: []string{
   707  //	        "photos/2006/February/",
   708  //	        "photos/2006/January/",
   709  //	    },
   710  //	}
   711  //
   712  // See http://goo.gl/YjQTc for details.
   713  func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) {
   714  	params := map[string][]string{
   715  		"prefix":    {prefix},
   716  		"delimiter": {delim},
   717  		"marker":    {marker},
   718  	}
   719  	if max != 0 {
   720  		params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)}
   721  	}
   722  	headers := map[string][]string{}
   723  	b.addTokenHeader(headers)
   724  
   725  	req := &request{
   726  		bucket:  b.Name,
   727  		params:  params,
   728  		headers: headers,
   729  	}
   730  	result = &ListResp{}
   731  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   732  		err = b.S3.query(context.Background(), req, result)
   733  		if !shouldRetry(err) {
   734  			break
   735  		}
   736  	}
   737  	if err != nil {
   738  		return nil, err
   739  	}
   740  	return result, nil
   741  }
   742  
   743  // The VersionsResp type holds the results of a list bucket Versions operation.
   744  type VersionsResp struct {
   745  	Name            string
   746  	Prefix          string
   747  	KeyMarker       string
   748  	VersionIDMarker string `xml:"VersionIdMarker"`
   749  	MaxKeys         int
   750  	Delimiter       string
   751  	IsTruncated     bool
   752  	Versions        []Version
   753  	CommonPrefixes  []string `xml:">Prefix"`
   754  }
   755  
   756  // The Version type represents an object version stored in an S3 bucket.
   757  type Version struct {
   758  	Key          string
   759  	VersionID    string `xml:"VersionId"`
   760  	IsLatest     bool
   761  	LastModified string
   762  	// ETag gives the hex-encoded MD5 sum of the contents,
   763  	// surrounded with double-quotes.
   764  	ETag         string
   765  	Size         int64
   766  	Owner        Owner
   767  	StorageClass string
   768  }
   769  
   770  func (b *Bucket) Versions(prefix, delim, keyMarker string, versionIDMarker string, max int) (result *VersionsResp, err error) {
   771  	params := map[string][]string{
   772  		"versions":  {""},
   773  		"prefix":    {prefix},
   774  		"delimiter": {delim},
   775  	}
   776  
   777  	if len(versionIDMarker) != 0 {
   778  		params["version-id-marker"] = []string{versionIDMarker}
   779  	}
   780  	if len(keyMarker) != 0 {
   781  		params["key-marker"] = []string{keyMarker}
   782  	}
   783  
   784  	if max != 0 {
   785  		params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)}
   786  	}
   787  	req := &request{
   788  		bucket: b.Name,
   789  		params: params,
   790  	}
   791  	result = &VersionsResp{}
   792  	for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); {
   793  		err = b.S3.query(context.Background(), req, result)
   794  		if !shouldRetry(err) {
   795  			break
   796  		}
   797  	}
   798  	if err != nil {
   799  		return nil, err
   800  	}
   801  	return result, nil
   802  }
   803  
   804  // Returns a mapping of all key names in this bucket to Key objects
   805  func (b *Bucket) GetBucketContents() (*map[string]Key, error) {
   806  	bucketContents := map[string]Key{}
   807  	prefix := ""
   808  	pathSeparator := ""
   809  	marker := ""
   810  	for {
   811  		contents, err := b.List(prefix, pathSeparator, marker, 1000)
   812  		if err != nil {
   813  			return &bucketContents, err
   814  		}
   815  		for _, key := range contents.Contents {
   816  			bucketContents[key.Key] = key
   817  		}
   818  		if contents.IsTruncated {
   819  			marker = contents.NextMarker
   820  		} else {
   821  			break
   822  		}
   823  	}
   824  
   825  	return &bucketContents, nil
   826  }
   827  
   828  // URL returns a non-signed URL that allows retrieving the
   829  // object at path. It only works if the object is publicly
   830  // readable (see SignedURL).
   831  func (b *Bucket) URL(path string) string {
   832  	req := &request{
   833  		bucket: b.Name,
   834  		path:   path,
   835  	}
   836  	err := b.S3.prepare(req)
   837  	if err != nil {
   838  		panic(err)
   839  	}
   840  	u, err := req.url()
   841  	if err != nil {
   842  		panic(err)
   843  	}
   844  	u.RawQuery = ""
   845  	return u.String()
   846  }
   847  
   848  // SignedURL returns a signed URL that allows anyone holding the URL
   849  // to retrieve the object at path. The signature is valid until expires.
   850  func (b *Bucket) SignedURL(path string, expires time.Time) string {
   851  	req := &request{
   852  		bucket: b.Name,
   853  		path:   path,
   854  		params: url.Values{"Expires": {strconv.FormatInt(expires.Unix(), 10)}},
   855  	}
   856  	err := b.S3.prepare(req)
   857  	if err != nil {
   858  		panic(err)
   859  	}
   860  	u, err := req.url()
   861  	if err != nil {
   862  		panic(err)
   863  	}
   864  	return u.String()
   865  }
   866  
   867  type request struct {
   868  	method   string
   869  	bucket   string
   870  	path     string
   871  	params   url.Values
   872  	headers  http.Header
   873  	baseurl  string
   874  	payload  io.Reader
   875  	prepared bool
   876  }
   877  
   878  func (req *request) url() (*url.URL, error) {
   879  	u, err := url.Parse(req.baseurl)
   880  	if err != nil {
   881  		return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err)
   882  	}
   883  	u.RawQuery = req.params.Encode()
   884  	u.Path = req.path
   885  	return u, nil
   886  }
   887  
   888  // query prepares and runs the req request.
   889  // If resp is not nil, the XML data contained in the response
   890  // body will be unmarshalled on it.
   891  func (s3 *S3) query(ctx context.Context, req *request, resp interface{}) error {
   892  	err := s3.prepare(req)
   893  	if err == nil {
   894  		var httpResponse *http.Response
   895  		httpResponse, err = s3.run(ctx, req, resp)
   896  		if resp == nil && httpResponse != nil {
   897  			httpResponse.Body.Close()
   898  		}
   899  	}
   900  	return err
   901  }
   902  
   903  // prepare sets up req to be delivered to S3.
   904  func (s3 *S3) prepare(req *request) error {
   905  	var signpath = req.path
   906  
   907  	if !req.prepared {
   908  		req.prepared = true
   909  		if req.method == "" {
   910  			req.method = "GET"
   911  		}
   912  		// Copy so they can be mutated without affecting on retries.
   913  		params := make(url.Values)
   914  		headers := make(http.Header)
   915  		for k, v := range req.params {
   916  			params[k] = v
   917  		}
   918  		for k, v := range req.headers {
   919  			headers[k] = v
   920  		}
   921  		req.params = params
   922  		req.headers = headers
   923  		if !strings.HasPrefix(req.path, "/") {
   924  			req.path = "/" + req.path
   925  		}
   926  		signpath = req.path
   927  		if req.bucket != "" {
   928  			req.baseurl = s3.Region.S3BucketEndpoint
   929  			if req.baseurl == "" {
   930  				// Use the path method to address the bucket.
   931  				req.baseurl = s3.Region.S3Endpoint
   932  				req.path = "/" + req.bucket + req.path
   933  			} else {
   934  				// Just in case, prevent injection.
   935  				if strings.ContainsAny(req.bucket, "/:@") {
   936  					return fmt.Errorf("bad S3 bucket: %q", req.bucket)
   937  				}
   938  				req.baseurl = strings.ReplaceAll(req.baseurl, "${bucket}", req.bucket)
   939  			}
   940  			signpath = "/" + req.bucket + signpath
   941  		}
   942  	}
   943  
   944  	// Always sign again as it's not clear how far the
   945  	// server has handled a previous attempt.
   946  	u, err := url.Parse(req.baseurl)
   947  	if err != nil {
   948  		return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err)
   949  	}
   950  	reqSignpathSpaceFix := (&url.URL{Path: signpath}).String()
   951  	req.headers["Host"] = []string{u.Host}
   952  	req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
   953  	return s3.sign(req.method, reqSignpathSpaceFix, req.params, req.headers)
   954  }
   955  
   956  // run sends req and returns the http response from the server.
   957  // If resp is not nil, the XML data contained in the response
   958  // body will be unmarshalled on it.
   959  func (s3 *S3) run(ctx context.Context, req *request, resp interface{}) (*http.Response, error) {
   960  	if debug {
   961  		log.Printf("Running S3 request: %#v", req)
   962  	}
   963  
   964  	u, err := req.url()
   965  	if err != nil {
   966  		return nil, err
   967  	}
   968  
   969  	hreq := &http.Request{
   970  		URL:        u,
   971  		Method:     req.method,
   972  		ProtoMajor: 1,
   973  		ProtoMinor: 1,
   974  		Close:      true,
   975  		Header:     req.headers,
   976  	}
   977  
   978  	if ctx != nil {
   979  		hreq = hreq.WithContext(ctx)
   980  	}
   981  
   982  	if v, ok := req.headers["Content-Length"]; ok {
   983  		hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64)
   984  		delete(req.headers, "Content-Length")
   985  	}
   986  	if req.payload != nil {
   987  		hreq.Body = io.NopCloser(req.payload)
   988  	}
   989  
   990  	if s3.client == nil {
   991  		s3.client = &http.Client{
   992  			Transport: &http.Transport{
   993  				Dial: func(netw, addr string) (c net.Conn, err error) {
   994  					c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout)
   995  					if err != nil {
   996  						return
   997  					}
   998  
   999  					var deadline time.Time
  1000  					if s3.RequestTimeout > 0 {
  1001  						deadline = time.Now().Add(s3.RequestTimeout)
  1002  						err := c.SetDeadline(deadline)
  1003  						if err != nil {
  1004  							return nil, err
  1005  						}
  1006  					}
  1007  
  1008  					if s3.ReadTimeout > 0 || s3.WriteTimeout > 0 {
  1009  						c = &ioTimeoutConn{
  1010  							TCPConn:         c.(*net.TCPConn),
  1011  							readTimeout:     s3.ReadTimeout,
  1012  							writeTimeout:    s3.WriteTimeout,
  1013  							requestDeadline: deadline,
  1014  						}
  1015  					}
  1016  					return
  1017  				},
  1018  				Proxy: libkb.MakeProxy(s3.G().Env),
  1019  			},
  1020  		}
  1021  	}
  1022  
  1023  	hresp, err := s3.client.Do(hreq)
  1024  	if err != nil {
  1025  		return nil, err
  1026  	}
  1027  	if debug {
  1028  		dump, _ := httputil.DumpResponse(hresp, true)
  1029  		log.Printf("} -> %s\n", dump)
  1030  	}
  1031  	if hresp.StatusCode != 200 && hresp.StatusCode != 204 && hresp.StatusCode != 206 {
  1032  		defer hresp.Body.Close()
  1033  		return nil, buildError(hresp)
  1034  	}
  1035  	if resp != nil {
  1036  		err = xml.NewDecoder(hresp.Body).Decode(resp)
  1037  		hresp.Body.Close()
  1038  		if debug {
  1039  			log.Printf("goamz.s3> decoded xml into %#v", resp)
  1040  		}
  1041  	}
  1042  	return hresp, err
  1043  }
  1044  
  1045  // Error represents an error in an operation with S3.
  1046  type Error struct {
  1047  	StatusCode int    // HTTP status code (200, 403, ...)
  1048  	Code       string // EC2 error code ("UnsupportedOperation", ...)
  1049  	Message    string // The human-oriented error message
  1050  	BucketName string
  1051  	RequestID  string `xml:"RequestId"`
  1052  	HostID     string `xml:"HostId"`
  1053  }
  1054  
  1055  func (e *Error) Error() string {
  1056  	return e.Message
  1057  }
  1058  
  1059  func buildError(r *http.Response) error {
  1060  	if debug {
  1061  		log.Printf("got error (status code %v)", r.StatusCode)
  1062  		data, err := io.ReadAll(r.Body)
  1063  		if err != nil {
  1064  			log.Printf("\tread error: %v", err)
  1065  		} else {
  1066  			log.Printf("\tdata:\n%s\n\n", data)
  1067  		}
  1068  		r.Body = io.NopCloser(bytes.NewBuffer(data))
  1069  	}
  1070  
  1071  	err := Error{}
  1072  
  1073  	// TODO return error if Unmarshal fails?
  1074  	decodeErr := xml.NewDecoder(r.Body).Decode(&err)
  1075  	if decodeErr != nil {
  1076  		log.Printf("\tdecodeErr error: %v", decodeErr)
  1077  	}
  1078  	r.Body.Close()
  1079  	err.StatusCode = r.StatusCode
  1080  	if err.Message == "" {
  1081  		err.Message = r.Status
  1082  	}
  1083  	if debug {
  1084  		log.Printf("err: %#v\n", err)
  1085  	}
  1086  	return &err
  1087  }
  1088  
  1089  func shouldRetry(err error) bool {
  1090  	if err == nil {
  1091  		return false
  1092  	}
  1093  	if e, ok := err.(*url.Error); ok {
  1094  		// Transport returns this string if it detects a write on a connection which
  1095  		// has already had an error
  1096  		if e.Err.Error() == "http: can't write HTTP request on broken connection" {
  1097  			return true
  1098  		}
  1099  		err = e.Err
  1100  	}
  1101  
  1102  	switch err {
  1103  	case io.ErrUnexpectedEOF, io.EOF:
  1104  		return true
  1105  	}
  1106  	switch e := err.(type) {
  1107  	case *net.DNSError:
  1108  		return true
  1109  	case *net.OpError:
  1110  		switch e.Op {
  1111  		case "read", "write", "WSARecv", "WSASend", "ConnectEx":
  1112  			return true
  1113  		}
  1114  	case *Error:
  1115  		switch e.Code {
  1116  		case "InternalError", "NoSuchUpload", "NoSuchBucket", "RequestTimeout":
  1117  			return true
  1118  		}
  1119  	// let's handle tls handshake timeout issues and similar temporary errors
  1120  	case net.Error:
  1121  		return e.Temporary() //nolint
  1122  	}
  1123  
  1124  	return false
  1125  }
  1126  
  1127  func hasCode(err error, code string) bool {
  1128  	s3err, ok := err.(*Error)
  1129  	return ok && s3err.Code == code
  1130  }
  1131  
  1132  // ioTimeoutConn is a net.Conn which sets a deadline for each Read or Write operation
  1133  type ioTimeoutConn struct {
  1134  	*net.TCPConn
  1135  	readTimeout     time.Duration
  1136  	writeTimeout    time.Duration
  1137  	requestDeadline time.Time
  1138  }
  1139  
  1140  func (c *ioTimeoutConn) deadline(timeout time.Duration) time.Time {
  1141  	dl := time.Now().Add(timeout)
  1142  	if c.requestDeadline.IsZero() || dl.Before(c.requestDeadline) {
  1143  		return dl
  1144  	}
  1145  
  1146  	return c.requestDeadline
  1147  }
  1148  
  1149  func (c *ioTimeoutConn) Read(b []byte) (int, error) {
  1150  	if c.readTimeout > 0 {
  1151  		err := c.TCPConn.SetReadDeadline(c.deadline(c.readTimeout))
  1152  		if err != nil {
  1153  			return 0, err
  1154  		}
  1155  	}
  1156  	return c.TCPConn.Read(b)
  1157  }
  1158  
  1159  func (c *ioTimeoutConn) Write(b []byte) (int, error) {
  1160  	if c.writeTimeout > 0 {
  1161  		err := c.TCPConn.SetWriteDeadline(c.deadline(c.writeTimeout))
  1162  		if err != nil {
  1163  			return 0, err
  1164  		}
  1165  	}
  1166  	return c.TCPConn.Write(b)
  1167  }