github.com/pquerna/agent@v2.1.8+incompatible/agent/s3_uploader.go (about) 1 package agent 2 3 import ( 4 "errors" 5 "fmt" 6 "io/ioutil" 7 "net/url" 8 "os" 9 "path/filepath" 10 "strings" 11 12 "github.com/AdRoll/goamz/s3" 13 "github.com/buildkite/agent/api" 14 "github.com/buildkite/agent/logger" 15 "github.com/buildkite/agent/mime" 16 ) 17 18 type S3Uploader struct { 19 // The destination which includes the S3 bucket name 20 // and the path. 21 // s3://my-bucket-name/foo/bar 22 Destination string 23 24 // Whether or not HTTP calls shoud be debugged 25 DebugHTTP bool 26 27 // The S3 Bucket we're uploading these files to 28 Bucket *s3.Bucket 29 } 30 31 func (u *S3Uploader) Setup(destination string, debugHTTP bool) error { 32 u.Destination = destination 33 u.DebugHTTP = debugHTTP 34 35 // Try to auth with S3 36 auth, err := awsS3Auth() 37 if err != nil { 38 return errors.New(fmt.Sprintf("Error creating AWS S3 authentication: %s", err.Error())) 39 } 40 41 // Try and get the region 42 region, err := awsS3Region() 43 if err != nil { 44 return err 45 } 46 47 logger.Debug("Authorizing S3 credentials and finding bucket `%s` in region `%s`...", u.bucketName(), region.Name) 48 49 // Find the bucket 50 s3 := s3.New(auth, region) 51 bucket := s3.Bucket(u.bucketName()) 52 53 // If the list doesn't return an error, then we've got our bucket 54 _, err = bucket.List("", "", "", 0) 55 if err != nil { 56 return errors.New("Could not find bucket `" + u.bucketName() + "` in region `" + region.Name + "` (" + err.Error() + ")") 57 } 58 59 u.Bucket = bucket 60 61 return nil 62 } 63 64 func (u *S3Uploader) URL(artifact *api.Artifact) string { 65 baseUrl := "https://" + u.bucketName() + ".s3.amazonaws.com" 66 if os.Getenv("BUILDKITE_S3_ACCESS_URL") != "" { 67 baseUrl = os.Getenv("BUILDKITE_S3_ACCESS_URL") 68 } 69 url, _ := url.Parse(baseUrl) 70 71 url.Path += u.artifactPath(artifact) 72 73 return url.String() 74 } 75 76 func (u *S3Uploader) Upload(artifact *api.Artifact) error { 77 permission := "public-read" 78 if os.Getenv("BUILDKITE_S3_ACL") != "" { 79 permission = os.Getenv("BUILDKITE_S3_ACL") 80 } else if os.Getenv("AWS_S3_ACL") != "" { 81 permission = os.Getenv("AWS_S3_ACL") 82 } 83 84 // The dirtiest validation method ever... 85 if permission != "private" && 86 permission != "public-read" && 87 permission != "public-read-write" && 88 permission != "authenticated-read" && 89 permission != "bucket-owner-read" && 90 permission != "bucket-owner-full-control" { 91 logger.Fatal("Invalid S3 ACL `%s`", permission) 92 } 93 94 Perms := s3.ACL(permission) 95 96 logger.Debug("Reading file \"%s\"", artifact.AbsolutePath) 97 data, err := ioutil.ReadFile(artifact.AbsolutePath) 98 if err != nil { 99 return errors.New("Failed to read file " + artifact.AbsolutePath + " (" + err.Error() + ")") 100 } 101 102 logger.Debug("Uploading \"%s\" to bucket with permission `%s`", u.artifactPath(artifact), permission) 103 err = u.Bucket.Put(u.artifactPath(artifact), data, u.mimeType(artifact), Perms, s3.Options{}) 104 if err != nil { 105 return errors.New(fmt.Sprintf("Failed to PUT file \"%s\" (%s)", u.artifactPath(artifact), err.Error())) 106 } 107 108 return nil 109 } 110 111 func (u *S3Uploader) artifactPath(artifact *api.Artifact) string { 112 parts := []string{u.bucketPath(), artifact.Path} 113 114 return strings.Join(parts, "/") 115 } 116 117 func (u *S3Uploader) bucketPath() string { 118 return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/") 119 } 120 121 func (u *S3Uploader) bucketName() string { 122 return u.destinationParts()[0] 123 } 124 125 func (u *S3Uploader) destinationParts() []string { 126 trimmed := strings.TrimPrefix(u.Destination, "s3://") 127 128 return strings.Split(trimmed, "/") 129 } 130 131 func (u *S3Uploader) mimeType(a *api.Artifact) string { 132 extension := filepath.Ext(a.Path) 133 mimeType := mime.TypeByExtension(extension) 134 135 if mimeType != "" { 136 return mimeType 137 } else { 138 return "binary/octet-stream" 139 } 140 }