github.com/alloyci/alloy-runner@v1.0.1-0.20180222164613-925503ccafd6/shells/cache.go (about)

     1  package shells
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/xml"
     6  	"io/ioutil"
     7  	"net/http"
     8  	"net/url"
     9  	"path"
    10  	"strconv"
    11  	"time"
    12  
    13  	"github.com/minio/minio-go"
    14  	"github.com/minio/minio-go/pkg/credentials"
    15  
    16  	"github.com/Sirupsen/logrus"
    17  	"gitlab.com/gitlab-org/gitlab-runner/common"
    18  )
    19  
    20  type bucketLocationTripper struct {
    21  	bucketLocation string
    22  }
    23  
    24  // The Minio Golang library always attempts to query the bucket location and
    25  // currently has no way of statically setting that value.  To avoid that
    26  // lookup, the Runner cache uses the library only to generate the URLs,
    27  // forgoing the library's API for uploading and downloading files. The custom
    28  // Roundtripper stubs out any network requests that would normally be made via
    29  // the library.
    30  func (b *bucketLocationTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
    31  	var buffer bytes.Buffer
    32  	xml.NewEncoder(&buffer).Encode(b.bucketLocation)
    33  	res = &http.Response{
    34  		StatusCode: http.StatusOK,
    35  		Body:       ioutil.NopCloser(&buffer),
    36  	}
    37  	return
    38  }
    39  
    40  func (b *bucketLocationTripper) CancelRequest(req *http.Request) {
    41  	// Do nothing
    42  }
    43  
    44  func getCacheObjectName(build *common.Build, cache *common.CacheConfig, key string) string {
    45  	if key == "" {
    46  		return ""
    47  	}
    48  	runnerSegment := ""
    49  	if !cache.Shared {
    50  		runnerSegment = path.Join("runner", build.Runner.ShortDescription())
    51  	}
    52  	return path.Join(cache.Path, runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key)
    53  }
    54  
    55  type fakeIAMCredentialsProvider interface {
    56  	credentials.Provider
    57  }
    58  
    59  var iamFactory = func() *credentials.Credentials {
    60  	return credentials.NewIAM("")
    61  }
    62  
    63  func getCacheStorageClient(cache *common.CacheConfig) (scl *minio.Client, err error) {
    64  	// If the server address or credentials aren't specified then use IAM
    65  	// instance profile credentials and talk to "real" S3.
    66  	if cache.ServerAddress == "" || cache.AccessKey == "" || cache.SecretKey == "" {
    67  		iam := iamFactory()
    68  		scl, err = minio.NewWithCredentials("s3.amazonaws.com", iam, true, "")
    69  	} else {
    70  		scl, err = minio.New(cache.ServerAddress, cache.AccessKey, cache.SecretKey, !cache.Insecure)
    71  	}
    72  	if err != nil {
    73  		logrus.Warningln(err)
    74  		return
    75  	}
    76  
    77  	scl.SetCustomTransport(&bucketLocationTripper{cache.BucketLocation})
    78  	return
    79  }
    80  
    81  func getS3DownloadURL(build *common.Build, key string) (url *url.URL) {
    82  	cache := build.Runner.Cache
    83  	objectName := getCacheObjectName(build, cache, key)
    84  	if objectName == "" {
    85  		return
    86  	}
    87  
    88  	scl, err := getCacheStorageClient(cache)
    89  	if err != nil {
    90  		logrus.Warningln(err)
    91  		return
    92  	}
    93  
    94  	url, err = scl.PresignedGetObject(cache.BucketName, objectName, time.Second*time.Duration(build.RunnerInfo.Timeout), nil)
    95  	if err != nil {
    96  		logrus.Warningln(err)
    97  		return
    98  	}
    99  	return
   100  }
   101  
   102  func getCacheDownloadURL(build *common.Build, key string) (url *url.URL) {
   103  	cache := build.Runner.Cache
   104  	if cache == nil {
   105  		return
   106  	}
   107  
   108  	switch cache.Type {
   109  	case "s3":
   110  		return getS3DownloadURL(build, key)
   111  	}
   112  	return
   113  }
   114  
   115  func getS3UploadURL(build *common.Build, key string) (url *url.URL) {
   116  	cache := build.Runner.Cache
   117  	objectName := getCacheObjectName(build, cache, key)
   118  	if objectName == "" {
   119  		return
   120  	}
   121  
   122  	scl, err := getCacheStorageClient(cache)
   123  	if err != nil {
   124  		logrus.Warningln(err)
   125  		return
   126  	}
   127  
   128  	url, err = scl.PresignedPutObject(cache.BucketName, objectName, time.Second*time.Duration(build.RunnerInfo.Timeout))
   129  	if err != nil {
   130  		logrus.Warningln(err)
   131  		return
   132  	}
   133  	return
   134  }
   135  
   136  func getCacheUploadURL(build *common.Build, key string) (url *url.URL) {
   137  	cache := build.Runner.Cache
   138  	if cache == nil {
   139  		return
   140  	}
   141  
   142  	switch cache.Type {
   143  	case "s3":
   144  		return getS3UploadURL(build, key)
   145  	}
   146  	return
   147  }