github.com/bshelton229/agent@v3.5.4+incompatible/agent/s3_uploader.go (about)

     1  package agent
     2  
     3  import (
     4  	"fmt"
     5  	"net/url"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  
    10  	"github.com/aws/aws-sdk-go/aws"
    11  	"github.com/aws/aws-sdk-go/service/s3"
    12  	"github.com/aws/aws-sdk-go/service/s3/s3manager"
    13  	"github.com/buildkite/agent/api"
    14  	"github.com/buildkite/agent/logger"
    15  	"github.com/buildkite/agent/mime"
    16  )
    17  
    18  type S3Uploader struct {
    19  	// The destination which includes the S3 bucket name and the path.
    20  	// e.g s3://my-bucket-name/foo/bar
    21  	Destination string
    22  
    23  	// Whether or not HTTP calls should be debugged
    24  	DebugHTTP bool
    25  
    26  	// The aws s3 client
    27  	s3Client *s3.S3
    28  }
    29  
    30  func (u *S3Uploader) Setup(destination string, debugHTTP bool) error {
    31  	u.Destination = destination
    32  	u.DebugHTTP = debugHTTP
    33  
    34  	// Initialize the s3 client, and authenticate it
    35  	s3Client, err := newS3Client(u.BucketName())
    36  	if err != nil {
    37  		return err
    38  	}
    39  
    40  	u.s3Client = s3Client
    41  	return nil
    42  }
    43  
    44  func (u *S3Uploader) URL(artifact *api.Artifact) string {
    45  	baseUrl := "https://" + u.BucketName() + ".s3.amazonaws.com"
    46  
    47  	if os.Getenv("BUILDKITE_S3_ACCESS_URL") != "" {
    48  		baseUrl = os.Getenv("BUILDKITE_S3_ACCESS_URL")
    49  	}
    50  
    51  	url, _ := url.Parse(baseUrl)
    52  
    53  	url.Path += u.artifactPath(artifact)
    54  
    55  	return url.String()
    56  }
    57  
    58  func (u *S3Uploader) Upload(artifact *api.Artifact) error {
    59  	permission := "public-read"
    60  	if os.Getenv("BUILDKITE_S3_ACL") != "" {
    61  		permission = os.Getenv("BUILDKITE_S3_ACL")
    62  	} else if os.Getenv("AWS_S3_ACL") != "" {
    63  		permission = os.Getenv("AWS_S3_ACL")
    64  	}
    65  
    66  	// The dirtiest validation method ever...
    67  	if permission != "private" &&
    68  		permission != "public-read" &&
    69  		permission != "public-read-write" &&
    70  		permission != "authenticated-read" &&
    71  		permission != "bucket-owner-read" &&
    72  		permission != "bucket-owner-full-control" {
    73  		return fmt.Errorf("Invalid S3 ACL `%s`", permission)
    74  	}
    75  
    76  	// Initialize the s3 client, and authenticate it
    77  	s3Client, err := newS3Client(u.BucketName())
    78  	if err != nil {
    79  		return err
    80  	}
    81  
    82  	// Create an uploader with the session and default options
    83  	uploader := s3manager.NewUploaderWithClient(s3Client)
    84  
    85  	// Open file from filesystem
    86  	logger.Debug("Reading file \"%s\"", artifact.AbsolutePath)
    87  	f, err := os.Open(artifact.AbsolutePath)
    88  	if err != nil {
    89  		return fmt.Errorf("failed to open file %q (%v)", artifact.AbsolutePath, err)
    90  	}
    91  
    92  	// Upload the file to S3.
    93  	logger.Debug("Uploading \"%s\" to bucket with permission `%s`", u.artifactPath(artifact), permission)
    94  	_, err = uploader.Upload(&s3manager.UploadInput{
    95  		Bucket:      aws.String(u.BucketName()),
    96  		Key:         aws.String(u.artifactPath(artifact)),
    97  		ContentType: aws.String(u.mimeType(artifact)),
    98  		ACL:         aws.String(permission),
    99  		Body:        f,
   100  	})
   101  
   102  	return err
   103  }
   104  
   105  func (u *S3Uploader) artifactPath(artifact *api.Artifact) string {
   106  	parts := []string{u.BucketPath(), artifact.Path}
   107  
   108  	return strings.Join(parts, "/")
   109  }
   110  
   111  func (u *S3Uploader) BucketPath() string {
   112  	return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/")
   113  }
   114  
   115  func (u *S3Uploader) BucketName() string {
   116  	return u.destinationParts()[0]
   117  }
   118  
   119  func (u *S3Uploader) destinationParts() []string {
   120  	trimmed := strings.TrimPrefix(u.Destination, "s3://")
   121  
   122  	return strings.Split(trimmed, "/")
   123  }
   124  
   125  func (u *S3Uploader) mimeType(a *api.Artifact) string {
   126  	extension := filepath.Ext(a.Path)
   127  	mimeType := mime.TypeByExtension(extension)
   128  
   129  	if mimeType != "" {
   130  		return mimeType
   131  	} else {
   132  		return "binary/octet-stream"
   133  	}
   134  }