github.com/pquerna/agent@v2.1.8+incompatible/agent/download.go (about) 1 package agent 2 3 import ( 4 "fmt" 5 "io" 6 "net/http" 7 "net/http/httputil" 8 "os" 9 "path/filepath" 10 "strings" 11 "time" 12 13 "github.com/buildkite/agent/logger" 14 "github.com/buildkite/agent/retry" 15 ) 16 17 type Download struct { 18 // The actual URL to get the file from 19 URL string 20 21 // The root directory of the download 22 Destination string 23 24 // The relative path that should be preserved in the download folder 25 Path string 26 27 // How many times should it retry the download before giving up 28 Retries int 29 30 // If failed responses should be dumped to the log 31 DebugHTTP bool 32 } 33 34 func (d Download) Start() error { 35 return retry.Do(func(s *retry.Stats) error { 36 err := d.try() 37 if err != nil { 38 logger.Warn("Error trying to download %s (%s) %s", d.URL, err, s) 39 } 40 return err 41 }, &retry.Config{Maximum: d.Retries, Interval: 5 * time.Second}) 42 } 43 44 func (d Download) try() error { 45 // If we're downloading a file with a path of "pkg/foo.txt" to a folder 46 // called "pkg", we should merge the two paths together. So, instead of it 47 // downloading to: destination/pkg/pkg/foo.txt, it will just download to 48 // destination/pkg/foo.txt 49 destinationPaths := strings.Split(d.Destination, string(os.PathSeparator)) 50 downloadPaths := strings.Split(d.Path, string(os.PathSeparator)) 51 52 for i := 0; i < len(downloadPaths); i += 100 { 53 // If the last part of the destination path matches 54 // this path in the download, then cut it out. 55 lastIndex := len(destinationPaths) - 1 56 57 // Break if we've gone too far. 58 if lastIndex == -1 { 59 break 60 } 61 62 lastPathInDestination := destinationPaths[lastIndex] 63 if lastPathInDestination == downloadPaths[i] { 64 destinationPaths = destinationPaths[:lastIndex] 65 } 66 } 67 68 finalizedDestination := strings.Join(destinationPaths, string(os.PathSeparator)) 69 70 targetFile := filepath.Join(finalizedDestination, d.Path) 71 targetDirectory, _ := filepath.Split(targetFile) 72 73 // Show a nice message that we're starting to download the file 74 logger.Debug("Downloading %s to %s", d.URL, targetFile) 75 76 // Start by downloading the file 77 response, err := http.Get(d.URL) 78 if err != nil { 79 return fmt.Errorf("Error while downloading %s (%T: %v)", d.URL, err, err) 80 } 81 defer response.Body.Close() 82 83 // Double check the status 84 if response.StatusCode/100 != 2 && response.StatusCode/100 != 3 { 85 if d.DebugHTTP { 86 responseDump, err := httputil.DumpResponse(response, true) 87 logger.Debug("\nERR: %s\n%s", err, string(responseDump)) 88 } 89 90 return &downloadError{response.Status} 91 } 92 93 // Now make the folder for our file 94 err = os.MkdirAll(targetDirectory, 0777) 95 if err != nil { 96 return fmt.Errorf("Failed to create folder for %s (%T: %v)", targetFile, err, err) 97 } 98 99 // Create a file to handle the file 100 fileBuffer, err := os.Create(targetFile) 101 if err != nil { 102 return fmt.Errorf("Failed to create file %s (%T: %v)", targetFile, err, err) 103 } 104 defer fileBuffer.Close() 105 106 // Copy the data to the file 107 bytes, err := io.Copy(fileBuffer, response.Body) 108 if err != nil { 109 return fmt.Errorf("Error when copying data %s (%T: %v)", d.URL, err, err) 110 } 111 112 logger.Info("Successfully downloaded \"%s\" %d bytes", d.Path, bytes) 113 114 return nil 115 } 116 117 type downloadError struct { 118 s string 119 } 120 121 func (e *downloadError) Error() string { 122 return e.s 123 }