github.com/stevenmatthewt/agent@v3.5.4+incompatible/agent/s3_downloader.go (about)

     1  package agent
     2  
     3  import (
     4  	"fmt"
     5  	"net/http"
     6  	"strings"
     7  	"time"
     8  
     9  	"github.com/aws/aws-sdk-go/aws"
    10  	"github.com/aws/aws-sdk-go/service/s3"
    11  )
    12  
    13  type S3Downloader struct {
    14  	// The S3 bucket name and the path, e.g s3://my-bucket-name/foo/bar
    15  	Bucket string
    16  
    17  	// The root directory of the download
    18  	Destination string
    19  
    20  	// The relative path that should be preserved in the download folder,
    21  	// also it's location in the bucket
    22  	Path string
    23  
    24  	// How many times should it retry the download before giving up
    25  	Retries int
    26  
    27  	// If failed responses should be dumped to the log
    28  	DebugHTTP bool
    29  }
    30  
    31  func (d S3Downloader) Start() error {
    32  	// Initialize the s3 client, and authenticate it
    33  	s3Client, err := newS3Client(d.BucketName())
    34  	if err != nil {
    35  		return err
    36  	}
    37  
    38  	req, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{
    39  		Bucket: aws.String(d.BucketName()),
    40  		Key:    aws.String(d.BucketFileLocation()),
    41  	})
    42  
    43  	signedURL, err := req.Presign(time.Hour)
    44  	if err != nil {
    45  		return fmt.Errorf("error pre-signing request: %v", err)
    46  	}
    47  
    48  	// We can now cheat and pass the URL onto our regular downloader
    49  	return Download{
    50  		Client:      *http.DefaultClient,
    51  		URL:         signedURL,
    52  		Path:        d.Path,
    53  		Destination: d.Destination,
    54  		Retries:     d.Retries,
    55  		DebugHTTP:   d.DebugHTTP,
    56  	}.Start()
    57  }
    58  
    59  func (d S3Downloader) BucketFileLocation() string {
    60  	if d.BucketPath() != "" {
    61  		return strings.TrimSuffix(d.BucketPath(), "/") + "/" + strings.TrimPrefix(d.Path, "/")
    62  	} else {
    63  		return d.Path
    64  	}
    65  }
    66  
    67  func (d S3Downloader) BucketPath() string {
    68  	return strings.Join(d.destinationParts()[1:len(d.destinationParts())], "/")
    69  }
    70  
    71  func (d S3Downloader) BucketName() string {
    72  	return d.destinationParts()[0]
    73  }
    74  
    75  func (d S3Downloader) destinationParts() []string {
    76  	trimmed := strings.TrimPrefix(d.Bucket, "s3://")
    77  
    78  	return strings.Split(trimmed, "/")
    79  }