github.com/bshelton229/agent@v3.5.4+incompatible/agent/s3_downloader.go (about) 1 package agent 2 3 import ( 4 "fmt" 5 "net/http" 6 "strings" 7 "time" 8 9 "github.com/aws/aws-sdk-go/aws" 10 "github.com/aws/aws-sdk-go/service/s3" 11 ) 12 13 type S3Downloader struct { 14 // The S3 bucket name and the path, e.g s3://my-bucket-name/foo/bar 15 Bucket string 16 17 // The root directory of the download 18 Destination string 19 20 // The relative path that should be preserved in the download folder, 21 // also it's location in the bucket 22 Path string 23 24 // How many times should it retry the download before giving up 25 Retries int 26 27 // If failed responses should be dumped to the log 28 DebugHTTP bool 29 } 30 31 func (d S3Downloader) Start() error { 32 // Initialize the s3 client, and authenticate it 33 s3Client, err := newS3Client(d.BucketName()) 34 if err != nil { 35 return err 36 } 37 38 req, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ 39 Bucket: aws.String(d.BucketName()), 40 Key: aws.String(d.BucketFileLocation()), 41 }) 42 43 signedURL, err := req.Presign(time.Hour) 44 if err != nil { 45 return fmt.Errorf("error pre-signing request: %v", err) 46 } 47 48 // We can now cheat and pass the URL onto our regular downloader 49 return Download{ 50 Client: *http.DefaultClient, 51 URL: signedURL, 52 Path: d.Path, 53 Destination: d.Destination, 54 Retries: d.Retries, 55 DebugHTTP: d.DebugHTTP, 56 }.Start() 57 } 58 59 func (d S3Downloader) BucketFileLocation() string { 60 if d.BucketPath() != "" { 61 return strings.TrimSuffix(d.BucketPath(), "/") + "/" + strings.TrimPrefix(d.Path, "/") 62 } else { 63 return d.Path 64 } 65 } 66 67 func (d S3Downloader) BucketPath() string { 68 return strings.Join(d.destinationParts()[1:len(d.destinationParts())], "/") 69 } 70 71 func (d S3Downloader) BucketName() string { 72 return d.destinationParts()[0] 73 } 74 75 func (d S3Downloader) destinationParts() []string { 76 trimmed := strings.TrimPrefix(d.Bucket, "s3://") 77 78 return strings.Split(trimmed, "/") 79 }