github.com/bshelton229/agent@v3.5.4+incompatible/agent/gs_uploader.go (about)

     1  package agent
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"io/ioutil"
     8  	"net/http"
     9  	"net/url"
    10  	"os"
    11  	"path/filepath"
    12  	"strings"
    13  
    14  	"github.com/buildkite/agent/api"
    15  	"github.com/buildkite/agent/logger"
    16  	"github.com/buildkite/agent/mime"
    17  	"golang.org/x/oauth2"
    18  	"golang.org/x/oauth2/google"
    19  	"google.golang.org/api/googleapi"
    20  	storage "google.golang.org/api/storage/v1"
    21  )
    22  
    23  type GSUploader struct {
    24  	// The destination which includes the GS bucket name and the path.
    25  	// gs://my-bucket-name/foo/bar
    26  	Destination string
    27  
    28  	// Whether or not HTTP calls shoud be debugged
    29  	DebugHTTP bool
    30  
    31  	// The GS service
    32  	Service *storage.Service
    33  }
    34  
    35  func (u *GSUploader) Setup(destination string, debugHTTP bool) error {
    36  	u.Destination = destination
    37  	u.DebugHTTP = debugHTTP
    38  
    39  	client, err := u.getClient(storage.DevstorageFullControlScope)
    40  	if err != nil {
    41  		return errors.New(fmt.Sprintf("Error creating Google Cloud Storage client: %v", err))
    42  	}
    43  	service, err := storage.New(client)
    44  	if err != nil {
    45  		return err
    46  	}
    47  	u.Service = service
    48  
    49  	return nil
    50  }
    51  
    52  func (u *GSUploader) URL(artifact *api.Artifact) string {
    53  	host := "storage.googleapis.com"
    54  	if os.Getenv("BUILDKITE_GCS_ACCESS_HOST") != "" {
    55  		host = os.Getenv("BUILDKITE_GCS_ACCESS_HOST")
    56  	}
    57  
    58  	var artifactURL = &url.URL{
    59  		Scheme: "https",
    60  		Host:   host,
    61  		Path:   u.BucketName() + "/" + u.artifactPath(artifact),
    62  	}
    63  	return artifactURL.String()
    64  }
    65  
    66  func (u *GSUploader) Upload(artifact *api.Artifact) error {
    67  	permission := os.Getenv("BUILDKITE_GS_ACL")
    68  
    69  	// The dirtiest validation method ever...
    70  	if permission != "" &&
    71  		permission != "authenticatedRead" &&
    72  		permission != "private" &&
    73  		permission != "projectPrivate" &&
    74  		permission != "publicRead" &&
    75  		permission != "publicReadWrite" {
    76  		logger.Fatal("Invalid GS ACL `%s`", permission)
    77  	}
    78  
    79  	if permission == "" {
    80  		logger.Debug("Uploading \"%s\" to bucket \"%s\" with default permission",
    81  			u.artifactPath(artifact), u.BucketName())
    82  	} else {
    83  		logger.Debug("Uploading \"%s\" to bucket \"%s\" with permission \"%s\"",
    84  			u.artifactPath(artifact), u.BucketName(), permission)
    85  	}
    86  	object := &storage.Object{
    87  		Name:               u.artifactPath(artifact),
    88  		ContentType:        u.mimeType(artifact),
    89  		ContentDisposition: u.contentDisposition(artifact),
    90  	}
    91  	file, err := os.Open(artifact.AbsolutePath)
    92  	if err != nil {
    93  		return errors.New(fmt.Sprintf("Failed to open file \"%q\" (%v)", artifact.AbsolutePath, err))
    94  	}
    95  	call := u.Service.Objects.Insert(u.BucketName(), object)
    96  	if permission != "" {
    97  		call = call.PredefinedAcl(permission)
    98  	}
    99  	if res, err := call.Media(file, googleapi.ContentType("")).Do(); err == nil {
   100  		logger.Debug("Created object %v at location %v\n\n", res.Name, res.SelfLink)
   101  	} else {
   102  		return errors.New(fmt.Sprintf("Failed to PUT file \"%s\" (%v)", u.artifactPath(artifact), err))
   103  	}
   104  
   105  	return nil
   106  }
   107  
   108  func (u *GSUploader) artifactPath(artifact *api.Artifact) string {
   109  	parts := []string{u.BucketPath(), artifact.Path}
   110  
   111  	return strings.Join(parts, "/")
   112  }
   113  
   114  func (u *GSUploader) BucketPath() string {
   115  	return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/")
   116  }
   117  
   118  func (u *GSUploader) BucketName() string {
   119  	return u.destinationParts()[0]
   120  }
   121  
   122  func (u *GSUploader) destinationParts() []string {
   123  	trimmed := strings.TrimPrefix(u.Destination, "gs://")
   124  
   125  	return strings.Split(trimmed, "/")
   126  }
   127  
   128  func (u *GSUploader) getClient(scope string) (*http.Client, error) {
   129  	if os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS") != "" {
   130  		data, err := ioutil.ReadFile(os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS"))
   131  		if err != nil {
   132  			return nil, err
   133  		}
   134  		conf, err := google.JWTConfigFromJSON(data, scope)
   135  		if err != nil {
   136  			return nil, err
   137  		}
   138  		return conf.Client(oauth2.NoContext), nil
   139  	}
   140  	return google.DefaultClient(context.Background(), scope)
   141  }
   142  
   143  func (u *GSUploader) mimeType(a *api.Artifact) string {
   144  	extension := filepath.Ext(a.Path)
   145  	mimeType := mime.TypeByExtension(extension)
   146  
   147  	if mimeType != "" {
   148  		return mimeType
   149  	} else {
   150  		return "binary/octet-stream"
   151  	}
   152  }
   153  
   154  func (u *GSUploader) contentDisposition(a *api.Artifact) string {
   155  	return fmt.Sprintf("inline; filename=\"%s\"", filepath.Base(a.Path))
   156  }