github.com/stevenmatthewt/agent@v3.5.4+incompatible/agent/download.go (about) 1 package agent 2 3 import ( 4 "fmt" 5 "io" 6 "net/http" 7 "net/http/httputil" 8 "os" 9 "path/filepath" 10 "strings" 11 "time" 12 13 "github.com/buildkite/agent/logger" 14 "github.com/buildkite/agent/retry" 15 ) 16 17 type Download struct { 18 // The HTTP client to use for downloading 19 Client http.Client 20 21 // The actual URL to get the file from 22 URL string 23 24 // The root directory of the download 25 Destination string 26 27 // The relative path that should be preserved in the download folder 28 Path string 29 30 // How many times should it retry the download before giving up 31 Retries int 32 33 // If failed responses should be dumped to the log 34 DebugHTTP bool 35 } 36 37 func (d Download) Start() error { 38 return retry.Do(func(s *retry.Stats) error { 39 err := d.try() 40 if err != nil { 41 logger.Warn("Error trying to download %s (%s) %s", d.URL, err, s) 42 } 43 return err 44 }, &retry.Config{Maximum: d.Retries, Interval: 5 * time.Second}) 45 } 46 47 func (d Download) try() error { 48 // If we're downloading a file with a path of "pkg/foo.txt" to a folder 49 // called "pkg", we should merge the two paths together. So, instead of it 50 // downloading to: destination/pkg/pkg/foo.txt, it will just download to 51 // destination/pkg/foo.txt 52 destinationPaths := strings.Split(d.Destination, string(os.PathSeparator)) 53 downloadPaths := strings.Split(d.Path, string(os.PathSeparator)) 54 55 for i := 0; i < len(downloadPaths); i += 100 { 56 // If the last part of the destination path matches 57 // this path in the download, then cut it out. 58 lastIndex := len(destinationPaths) - 1 59 60 // Break if we've gone too far. 61 if lastIndex == -1 { 62 break 63 } 64 65 lastPathInDestination := destinationPaths[lastIndex] 66 if lastPathInDestination == downloadPaths[i] { 67 destinationPaths = destinationPaths[:lastIndex] 68 } 69 } 70 71 finalizedDestination := strings.Join(destinationPaths, string(os.PathSeparator)) 72 73 targetFile := filepath.Join(finalizedDestination, d.Path) 74 targetDirectory, _ := filepath.Split(targetFile) 75 76 // Show a nice message that we're starting to download the file 77 logger.Debug("Downloading %s to %s", d.URL, targetFile) 78 79 // Start by downloading the file 80 response, err := d.Client.Get(d.URL) 81 if err != nil { 82 return fmt.Errorf("Error while downloading %s (%T: %v)", d.URL, err, err) 83 } 84 defer response.Body.Close() 85 86 // Double check the status 87 if response.StatusCode/100 != 2 && response.StatusCode/100 != 3 { 88 if d.DebugHTTP { 89 responseDump, err := httputil.DumpResponse(response, true) 90 logger.Debug("\nERR: %s\n%s", err, string(responseDump)) 91 } 92 93 return &downloadError{response.Status} 94 } 95 96 // Now make the folder for our file 97 err = os.MkdirAll(targetDirectory, 0777) 98 if err != nil { 99 return fmt.Errorf("Failed to create folder for %s (%T: %v)", targetFile, err, err) 100 } 101 102 // Create a file to handle the file 103 fileBuffer, err := os.Create(targetFile) 104 if err != nil { 105 return fmt.Errorf("Failed to create file %s (%T: %v)", targetFile, err, err) 106 } 107 defer fileBuffer.Close() 108 109 // Copy the data to the file 110 bytes, err := io.Copy(fileBuffer, response.Body) 111 if err != nil { 112 return fmt.Errorf("Error when copying data %s (%T: %v)", d.URL, err, err) 113 } 114 115 logger.Info("Successfully downloaded \"%s\" %d bytes", d.Path, bytes) 116 117 return nil 118 } 119 120 type downloadError struct { 121 s string 122 } 123 124 func (e *downloadError) Error() string { 125 return e.s 126 }