github.com/pquerna/agent@v2.1.8+incompatible/agent/s3_downloader.go (about)

     1  package agent
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"strings"
     7  	"time"
     8  
     9  	"github.com/AdRoll/goamz/s3"
    10  	"github.com/buildkite/agent/logger"
    11  )
    12  
    13  type S3Downloader struct {
    14  	// The name of the bucket
    15  	Bucket string
    16  
    17  	// The root directory of the download
    18  	Destination string
    19  
    20  	// The relative path that should be preserved in the download folder,
    21  	// also it's location in the bucket
    22  	Path string
    23  
    24  	// How many times should it retry the download before giving up
    25  	Retries int
    26  
    27  	// If failed responses should be dumped to the log
    28  	DebugHTTP bool
    29  }
    30  
    31  func (d S3Downloader) Start() error {
    32  	// Try to auth with S3
    33  	auth, err := awsS3Auth()
    34  	if err != nil {
    35  		return errors.New(fmt.Sprintf("Error creating AWS S3 authentication: %s", err.Error()))
    36  	}
    37  
    38  	// Try and get the region
    39  	region, err := awsS3Region()
    40  	if err != nil {
    41  		return err
    42  	}
    43  
    44  	// Split apart the bucket
    45  	bucketParts := strings.Split(strings.TrimPrefix(d.Bucket, "s3://"), "/")
    46  	bucketName := bucketParts[0]
    47  	bucketPath := strings.Join(bucketParts[1:len(bucketParts)], "/")
    48  
    49  	logger.Debug("Authorizing S3 credentials and finding bucket `%s` in region `%s`...", bucketName, region.Name)
    50  
    51  	// Find the bucket
    52  	s3 := s3.New(auth, region)
    53  	bucket := s3.Bucket(bucketName)
    54  
    55  	// If the list doesn't return an error, then we've got our bucket
    56  	_, err = bucket.List("", "", "", 0)
    57  	if err != nil {
    58  		return errors.New("Could not find bucket `" + bucketName + "` in region `" + region.Name + "` (" + err.Error() + ")")
    59  	}
    60  
    61  	// Create the location of the file
    62  	var s3Location string
    63  	if bucketPath != "" {
    64  		s3Location = strings.TrimRight(bucketPath, "/") + "/" + strings.TrimLeft(d.Path, "/")
    65  	} else {
    66  		s3Location = d.Path
    67  	}
    68  
    69  	// Generate a Signed URL
    70  	signedURL := bucket.SignedURL(s3Location, time.Now().Add(time.Hour))
    71  
    72  	// We can now cheat and pass the URL onto our regular downloader
    73  	return Download{
    74  		URL:         signedURL,
    75  		Path:        d.Path,
    76  		Destination: d.Destination,
    77  		Retries:     d.Retries,
    78  		DebugHTTP:   d.DebugHTTP,
    79  	}.Start()
    80  }