github.com/stevenmatthewt/agent@v3.5.4+incompatible/agent/s3_uploader.go (about) 1 package agent 2 3 import ( 4 "fmt" 5 "net/url" 6 "os" 7 "path/filepath" 8 "strings" 9 10 "github.com/aws/aws-sdk-go/aws" 11 "github.com/aws/aws-sdk-go/service/s3" 12 "github.com/aws/aws-sdk-go/service/s3/s3manager" 13 "github.com/buildkite/agent/api" 14 "github.com/buildkite/agent/logger" 15 "github.com/buildkite/agent/mime" 16 ) 17 18 type S3Uploader struct { 19 // The destination which includes the S3 bucket name and the path. 20 // e.g s3://my-bucket-name/foo/bar 21 Destination string 22 23 // Whether or not HTTP calls should be debugged 24 DebugHTTP bool 25 26 // The aws s3 client 27 s3Client *s3.S3 28 } 29 30 func (u *S3Uploader) Setup(destination string, debugHTTP bool) error { 31 u.Destination = destination 32 u.DebugHTTP = debugHTTP 33 34 // Initialize the s3 client, and authenticate it 35 s3Client, err := newS3Client(u.BucketName()) 36 if err != nil { 37 return err 38 } 39 40 u.s3Client = s3Client 41 return nil 42 } 43 44 func (u *S3Uploader) URL(artifact *api.Artifact) string { 45 baseUrl := "https://" + u.BucketName() + ".s3.amazonaws.com" 46 47 if os.Getenv("BUILDKITE_S3_ACCESS_URL") != "" { 48 baseUrl = os.Getenv("BUILDKITE_S3_ACCESS_URL") 49 } 50 51 url, _ := url.Parse(baseUrl) 52 53 url.Path += u.artifactPath(artifact) 54 55 return url.String() 56 } 57 58 func (u *S3Uploader) Upload(artifact *api.Artifact) error { 59 permission := "public-read" 60 if os.Getenv("BUILDKITE_S3_ACL") != "" { 61 permission = os.Getenv("BUILDKITE_S3_ACL") 62 } else if os.Getenv("AWS_S3_ACL") != "" { 63 permission = os.Getenv("AWS_S3_ACL") 64 } 65 66 // The dirtiest validation method ever... 67 if permission != "private" && 68 permission != "public-read" && 69 permission != "public-read-write" && 70 permission != "authenticated-read" && 71 permission != "bucket-owner-read" && 72 permission != "bucket-owner-full-control" { 73 return fmt.Errorf("Invalid S3 ACL `%s`", permission) 74 } 75 76 // Initialize the s3 client, and authenticate it 77 s3Client, err := newS3Client(u.BucketName()) 78 if err != nil { 79 return err 80 } 81 82 // Create an uploader with the session and default options 83 uploader := s3manager.NewUploaderWithClient(s3Client) 84 85 // Open file from filesystem 86 logger.Debug("Reading file \"%s\"", artifact.AbsolutePath) 87 f, err := os.Open(artifact.AbsolutePath) 88 if err != nil { 89 return fmt.Errorf("failed to open file %q (%v)", artifact.AbsolutePath, err) 90 } 91 92 // Upload the file to S3. 93 logger.Debug("Uploading \"%s\" to bucket with permission `%s`", u.artifactPath(artifact), permission) 94 _, err = uploader.Upload(&s3manager.UploadInput{ 95 Bucket: aws.String(u.BucketName()), 96 Key: aws.String(u.artifactPath(artifact)), 97 ContentType: aws.String(u.mimeType(artifact)), 98 ACL: aws.String(permission), 99 Body: f, 100 }) 101 102 return err 103 } 104 105 func (u *S3Uploader) artifactPath(artifact *api.Artifact) string { 106 parts := []string{u.BucketPath(), artifact.Path} 107 108 return strings.Join(parts, "/") 109 } 110 111 func (u *S3Uploader) BucketPath() string { 112 return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/") 113 } 114 115 func (u *S3Uploader) BucketName() string { 116 return u.destinationParts()[0] 117 } 118 119 func (u *S3Uploader) destinationParts() []string { 120 trimmed := strings.TrimPrefix(u.Destination, "s3://") 121 122 return strings.Split(trimmed, "/") 123 } 124 125 func (u *S3Uploader) mimeType(a *api.Artifact) string { 126 extension := filepath.Ext(a.Path) 127 mimeType := mime.TypeByExtension(extension) 128 129 if mimeType != "" { 130 return mimeType 131 } else { 132 return "binary/octet-stream" 133 } 134 }