github.com/bshelton229/agent@v3.5.4+incompatible/agent/gs_uploader.go (about) 1 package agent 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "io/ioutil" 8 "net/http" 9 "net/url" 10 "os" 11 "path/filepath" 12 "strings" 13 14 "github.com/buildkite/agent/api" 15 "github.com/buildkite/agent/logger" 16 "github.com/buildkite/agent/mime" 17 "golang.org/x/oauth2" 18 "golang.org/x/oauth2/google" 19 "google.golang.org/api/googleapi" 20 storage "google.golang.org/api/storage/v1" 21 ) 22 23 type GSUploader struct { 24 // The destination which includes the GS bucket name and the path. 25 // gs://my-bucket-name/foo/bar 26 Destination string 27 28 // Whether or not HTTP calls shoud be debugged 29 DebugHTTP bool 30 31 // The GS service 32 Service *storage.Service 33 } 34 35 func (u *GSUploader) Setup(destination string, debugHTTP bool) error { 36 u.Destination = destination 37 u.DebugHTTP = debugHTTP 38 39 client, err := u.getClient(storage.DevstorageFullControlScope) 40 if err != nil { 41 return errors.New(fmt.Sprintf("Error creating Google Cloud Storage client: %v", err)) 42 } 43 service, err := storage.New(client) 44 if err != nil { 45 return err 46 } 47 u.Service = service 48 49 return nil 50 } 51 52 func (u *GSUploader) URL(artifact *api.Artifact) string { 53 host := "storage.googleapis.com" 54 if os.Getenv("BUILDKITE_GCS_ACCESS_HOST") != "" { 55 host = os.Getenv("BUILDKITE_GCS_ACCESS_HOST") 56 } 57 58 var artifactURL = &url.URL{ 59 Scheme: "https", 60 Host: host, 61 Path: u.BucketName() + "/" + u.artifactPath(artifact), 62 } 63 return artifactURL.String() 64 } 65 66 func (u *GSUploader) Upload(artifact *api.Artifact) error { 67 permission := os.Getenv("BUILDKITE_GS_ACL") 68 69 // The dirtiest validation method ever... 70 if permission != "" && 71 permission != "authenticatedRead" && 72 permission != "private" && 73 permission != "projectPrivate" && 74 permission != "publicRead" && 75 permission != "publicReadWrite" { 76 logger.Fatal("Invalid GS ACL `%s`", permission) 77 } 78 79 if permission == "" { 80 logger.Debug("Uploading \"%s\" to bucket \"%s\" with default permission", 81 u.artifactPath(artifact), u.BucketName()) 82 } else { 83 logger.Debug("Uploading \"%s\" to bucket \"%s\" with permission \"%s\"", 84 u.artifactPath(artifact), u.BucketName(), permission) 85 } 86 object := &storage.Object{ 87 Name: u.artifactPath(artifact), 88 ContentType: u.mimeType(artifact), 89 ContentDisposition: u.contentDisposition(artifact), 90 } 91 file, err := os.Open(artifact.AbsolutePath) 92 if err != nil { 93 return errors.New(fmt.Sprintf("Failed to open file \"%q\" (%v)", artifact.AbsolutePath, err)) 94 } 95 call := u.Service.Objects.Insert(u.BucketName(), object) 96 if permission != "" { 97 call = call.PredefinedAcl(permission) 98 } 99 if res, err := call.Media(file, googleapi.ContentType("")).Do(); err == nil { 100 logger.Debug("Created object %v at location %v\n\n", res.Name, res.SelfLink) 101 } else { 102 return errors.New(fmt.Sprintf("Failed to PUT file \"%s\" (%v)", u.artifactPath(artifact), err)) 103 } 104 105 return nil 106 } 107 108 func (u *GSUploader) artifactPath(artifact *api.Artifact) string { 109 parts := []string{u.BucketPath(), artifact.Path} 110 111 return strings.Join(parts, "/") 112 } 113 114 func (u *GSUploader) BucketPath() string { 115 return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/") 116 } 117 118 func (u *GSUploader) BucketName() string { 119 return u.destinationParts()[0] 120 } 121 122 func (u *GSUploader) destinationParts() []string { 123 trimmed := strings.TrimPrefix(u.Destination, "gs://") 124 125 return strings.Split(trimmed, "/") 126 } 127 128 func (u *GSUploader) getClient(scope string) (*http.Client, error) { 129 if os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS") != "" { 130 data, err := ioutil.ReadFile(os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS")) 131 if err != nil { 132 return nil, err 133 } 134 conf, err := google.JWTConfigFromJSON(data, scope) 135 if err != nil { 136 return nil, err 137 } 138 return conf.Client(oauth2.NoContext), nil 139 } 140 return google.DefaultClient(context.Background(), scope) 141 } 142 143 func (u *GSUploader) mimeType(a *api.Artifact) string { 144 extension := filepath.Ext(a.Path) 145 mimeType := mime.TypeByExtension(extension) 146 147 if mimeType != "" { 148 return mimeType 149 } else { 150 return "binary/octet-stream" 151 } 152 } 153 154 func (u *GSUploader) contentDisposition(a *api.Artifact) string { 155 return fmt.Sprintf("inline; filename=\"%s\"", filepath.Base(a.Path)) 156 }