github.com/pquerna/agent@v2.1.8+incompatible/agent/s3_uploader.go (about)

     1  package agent
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"net/url"
     8  	"os"
     9  	"path/filepath"
    10  	"strings"
    11  
    12  	"github.com/AdRoll/goamz/s3"
    13  	"github.com/buildkite/agent/api"
    14  	"github.com/buildkite/agent/logger"
    15  	"github.com/buildkite/agent/mime"
    16  )
    17  
    18  type S3Uploader struct {
    19  	// The destination which includes the S3 bucket name
    20  	// and the path.
    21  	// s3://my-bucket-name/foo/bar
    22  	Destination string
    23  
    24  	// Whether or not HTTP calls shoud be debugged
    25  	DebugHTTP bool
    26  
    27  	// The S3 Bucket we're uploading these files to
    28  	Bucket *s3.Bucket
    29  }
    30  
    31  func (u *S3Uploader) Setup(destination string, debugHTTP bool) error {
    32  	u.Destination = destination
    33  	u.DebugHTTP = debugHTTP
    34  
    35  	// Try to auth with S3
    36  	auth, err := awsS3Auth()
    37  	if err != nil {
    38  		return errors.New(fmt.Sprintf("Error creating AWS S3 authentication: %s", err.Error()))
    39  	}
    40  
    41  	// Try and get the region
    42  	region, err := awsS3Region()
    43  	if err != nil {
    44  		return err
    45  	}
    46  
    47  	logger.Debug("Authorizing S3 credentials and finding bucket `%s` in region `%s`...", u.bucketName(), region.Name)
    48  
    49  	// Find the bucket
    50  	s3 := s3.New(auth, region)
    51  	bucket := s3.Bucket(u.bucketName())
    52  
    53  	// If the list doesn't return an error, then we've got our bucket
    54  	_, err = bucket.List("", "", "", 0)
    55  	if err != nil {
    56  		return errors.New("Could not find bucket `" + u.bucketName() + "` in region `" + region.Name + "` (" + err.Error() + ")")
    57  	}
    58  
    59  	u.Bucket = bucket
    60  
    61  	return nil
    62  }
    63  
    64  func (u *S3Uploader) URL(artifact *api.Artifact) string {
    65  	baseUrl := "https://" + u.bucketName() + ".s3.amazonaws.com"
    66  	if os.Getenv("BUILDKITE_S3_ACCESS_URL") != "" {
    67  		baseUrl = os.Getenv("BUILDKITE_S3_ACCESS_URL")
    68  	}
    69  	url, _ := url.Parse(baseUrl)
    70  
    71  	url.Path += u.artifactPath(artifact)
    72  
    73  	return url.String()
    74  }
    75  
    76  func (u *S3Uploader) Upload(artifact *api.Artifact) error {
    77  	permission := "public-read"
    78  	if os.Getenv("BUILDKITE_S3_ACL") != "" {
    79  		permission = os.Getenv("BUILDKITE_S3_ACL")
    80  	} else if os.Getenv("AWS_S3_ACL") != "" {
    81  		permission = os.Getenv("AWS_S3_ACL")
    82  	}
    83  
    84  	// The dirtiest validation method ever...
    85  	if permission != "private" &&
    86  		permission != "public-read" &&
    87  		permission != "public-read-write" &&
    88  		permission != "authenticated-read" &&
    89  		permission != "bucket-owner-read" &&
    90  		permission != "bucket-owner-full-control" {
    91  		logger.Fatal("Invalid S3 ACL `%s`", permission)
    92  	}
    93  
    94  	Perms := s3.ACL(permission)
    95  
    96  	logger.Debug("Reading file \"%s\"", artifact.AbsolutePath)
    97  	data, err := ioutil.ReadFile(artifact.AbsolutePath)
    98  	if err != nil {
    99  		return errors.New("Failed to read file " + artifact.AbsolutePath + " (" + err.Error() + ")")
   100  	}
   101  
   102  	logger.Debug("Uploading \"%s\" to bucket with permission `%s`", u.artifactPath(artifact), permission)
   103  	err = u.Bucket.Put(u.artifactPath(artifact), data, u.mimeType(artifact), Perms, s3.Options{})
   104  	if err != nil {
   105  		return errors.New(fmt.Sprintf("Failed to PUT file \"%s\" (%s)", u.artifactPath(artifact), err.Error()))
   106  	}
   107  
   108  	return nil
   109  }
   110  
   111  func (u *S3Uploader) artifactPath(artifact *api.Artifact) string {
   112  	parts := []string{u.bucketPath(), artifact.Path}
   113  
   114  	return strings.Join(parts, "/")
   115  }
   116  
   117  func (u *S3Uploader) bucketPath() string {
   118  	return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/")
   119  }
   120  
   121  func (u *S3Uploader) bucketName() string {
   122  	return u.destinationParts()[0]
   123  }
   124  
   125  func (u *S3Uploader) destinationParts() []string {
   126  	trimmed := strings.TrimPrefix(u.Destination, "s3://")
   127  
   128  	return strings.Split(trimmed, "/")
   129  }
   130  
   131  func (u *S3Uploader) mimeType(a *api.Artifact) string {
   132  	extension := filepath.Ext(a.Path)
   133  	mimeType := mime.TypeByExtension(extension)
   134  
   135  	if mimeType != "" {
   136  		return mimeType
   137  	} else {
   138  		return "binary/octet-stream"
   139  	}
   140  }