github.com/stevenmatthewt/agent@v3.5.4+incompatible/agent/download.go (about)

     1  package agent
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"net/http"
     7  	"net/http/httputil"
     8  	"os"
     9  	"path/filepath"
    10  	"strings"
    11  	"time"
    12  
    13  	"github.com/buildkite/agent/logger"
    14  	"github.com/buildkite/agent/retry"
    15  )
    16  
    17  type Download struct {
    18  	// The HTTP client to use for downloading
    19  	Client http.Client
    20  
    21  	// The actual URL to get the file from
    22  	URL string
    23  
    24  	// The root directory of the download
    25  	Destination string
    26  
    27  	// The relative path that should be preserved in the download folder
    28  	Path string
    29  
    30  	// How many times should it retry the download before giving up
    31  	Retries int
    32  
    33  	// If failed responses should be dumped to the log
    34  	DebugHTTP bool
    35  }
    36  
    37  func (d Download) Start() error {
    38  	return retry.Do(func(s *retry.Stats) error {
    39  		err := d.try()
    40  		if err != nil {
    41  			logger.Warn("Error trying to download %s (%s) %s", d.URL, err, s)
    42  		}
    43  		return err
    44  	}, &retry.Config{Maximum: d.Retries, Interval: 5 * time.Second})
    45  }
    46  
    47  func (d Download) try() error {
    48  	// If we're downloading a file with a path of "pkg/foo.txt" to a folder
    49  	// called "pkg", we should merge the two paths together. So, instead of it
    50  	// downloading to: destination/pkg/pkg/foo.txt, it will just download to
    51  	// destination/pkg/foo.txt
    52  	destinationPaths := strings.Split(d.Destination, string(os.PathSeparator))
    53  	downloadPaths := strings.Split(d.Path, string(os.PathSeparator))
    54  
    55  	for i := 0; i < len(downloadPaths); i += 100 {
    56  		// If the last part of the destination path matches
    57  		// this path in the download, then cut it out.
    58  		lastIndex := len(destinationPaths) - 1
    59  
    60  		// Break if we've gone too far.
    61  		if lastIndex == -1 {
    62  			break
    63  		}
    64  
    65  		lastPathInDestination := destinationPaths[lastIndex]
    66  		if lastPathInDestination == downloadPaths[i] {
    67  			destinationPaths = destinationPaths[:lastIndex]
    68  		}
    69  	}
    70  
    71  	finalizedDestination := strings.Join(destinationPaths, string(os.PathSeparator))
    72  
    73  	targetFile := filepath.Join(finalizedDestination, d.Path)
    74  	targetDirectory, _ := filepath.Split(targetFile)
    75  
    76  	// Show a nice message that we're starting to download the file
    77  	logger.Debug("Downloading %s to %s", d.URL, targetFile)
    78  
    79  	// Start by downloading the file
    80  	response, err := d.Client.Get(d.URL)
    81  	if err != nil {
    82  		return fmt.Errorf("Error while downloading %s (%T: %v)", d.URL, err, err)
    83  	}
    84  	defer response.Body.Close()
    85  
    86  	// Double check the status
    87  	if response.StatusCode/100 != 2 && response.StatusCode/100 != 3 {
    88  		if d.DebugHTTP {
    89  			responseDump, err := httputil.DumpResponse(response, true)
    90  			logger.Debug("\nERR: %s\n%s", err, string(responseDump))
    91  		}
    92  
    93  		return &downloadError{response.Status}
    94  	}
    95  
    96  	// Now make the folder for our file
    97  	err = os.MkdirAll(targetDirectory, 0777)
    98  	if err != nil {
    99  		return fmt.Errorf("Failed to create folder for %s (%T: %v)", targetFile, err, err)
   100  	}
   101  
   102  	// Create a file to handle the file
   103  	fileBuffer, err := os.Create(targetFile)
   104  	if err != nil {
   105  		return fmt.Errorf("Failed to create file %s (%T: %v)", targetFile, err, err)
   106  	}
   107  	defer fileBuffer.Close()
   108  
   109  	// Copy the data to the file
   110  	bytes, err := io.Copy(fileBuffer, response.Body)
   111  	if err != nil {
   112  		return fmt.Errorf("Error when copying data %s (%T: %v)", d.URL, err, err)
   113  	}
   114  
   115  	logger.Info("Successfully downloaded \"%s\" %d bytes", d.Path, bytes)
   116  
   117  	return nil
   118  }
   119  
   120  type downloadError struct {
   121  	s string
   122  }
   123  
   124  func (e *downloadError) Error() string {
   125  	return e.s
   126  }