github.com/pquerna/agent@v2.1.8+incompatible/agent/download.go (about)

     1  package agent
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"net/http"
     7  	"net/http/httputil"
     8  	"os"
     9  	"path/filepath"
    10  	"strings"
    11  	"time"
    12  
    13  	"github.com/buildkite/agent/logger"
    14  	"github.com/buildkite/agent/retry"
    15  )
    16  
    17  type Download struct {
    18  	// The actual URL to get the file from
    19  	URL string
    20  
    21  	// The root directory of the download
    22  	Destination string
    23  
    24  	// The relative path that should be preserved in the download folder
    25  	Path string
    26  
    27  	// How many times should it retry the download before giving up
    28  	Retries int
    29  
    30  	// If failed responses should be dumped to the log
    31  	DebugHTTP bool
    32  }
    33  
    34  func (d Download) Start() error {
    35  	return retry.Do(func(s *retry.Stats) error {
    36  		err := d.try()
    37  		if err != nil {
    38  			logger.Warn("Error trying to download %s (%s) %s", d.URL, err, s)
    39  		}
    40  		return err
    41  	}, &retry.Config{Maximum: d.Retries, Interval: 5 * time.Second})
    42  }
    43  
    44  func (d Download) try() error {
    45  	// If we're downloading a file with a path of "pkg/foo.txt" to a folder
    46  	// called "pkg", we should merge the two paths together. So, instead of it
    47  	// downloading to: destination/pkg/pkg/foo.txt, it will just download to
    48  	// destination/pkg/foo.txt
    49  	destinationPaths := strings.Split(d.Destination, string(os.PathSeparator))
    50  	downloadPaths := strings.Split(d.Path, string(os.PathSeparator))
    51  
    52  	for i := 0; i < len(downloadPaths); i += 100 {
    53  		// If the last part of the destination path matches
    54  		// this path in the download, then cut it out.
    55  		lastIndex := len(destinationPaths) - 1
    56  
    57  		// Break if we've gone too far.
    58  		if lastIndex == -1 {
    59  			break
    60  		}
    61  
    62  		lastPathInDestination := destinationPaths[lastIndex]
    63  		if lastPathInDestination == downloadPaths[i] {
    64  			destinationPaths = destinationPaths[:lastIndex]
    65  		}
    66  	}
    67  
    68  	finalizedDestination := strings.Join(destinationPaths, string(os.PathSeparator))
    69  
    70  	targetFile := filepath.Join(finalizedDestination, d.Path)
    71  	targetDirectory, _ := filepath.Split(targetFile)
    72  
    73  	// Show a nice message that we're starting to download the file
    74  	logger.Debug("Downloading %s to %s", d.URL, targetFile)
    75  
    76  	// Start by downloading the file
    77  	response, err := http.Get(d.URL)
    78  	if err != nil {
    79  		return fmt.Errorf("Error while downloading %s (%T: %v)", d.URL, err, err)
    80  	}
    81  	defer response.Body.Close()
    82  
    83  	// Double check the status
    84  	if response.StatusCode/100 != 2 && response.StatusCode/100 != 3 {
    85  		if d.DebugHTTP {
    86  			responseDump, err := httputil.DumpResponse(response, true)
    87  			logger.Debug("\nERR: %s\n%s", err, string(responseDump))
    88  		}
    89  
    90  		return &downloadError{response.Status}
    91  	}
    92  
    93  	// Now make the folder for our file
    94  	err = os.MkdirAll(targetDirectory, 0777)
    95  	if err != nil {
    96  		return fmt.Errorf("Failed to create folder for %s (%T: %v)", targetFile, err, err)
    97  	}
    98  
    99  	// Create a file to handle the file
   100  	fileBuffer, err := os.Create(targetFile)
   101  	if err != nil {
   102  		return fmt.Errorf("Failed to create file %s (%T: %v)", targetFile, err, err)
   103  	}
   104  	defer fileBuffer.Close()
   105  
   106  	// Copy the data to the file
   107  	bytes, err := io.Copy(fileBuffer, response.Body)
   108  	if err != nil {
   109  		return fmt.Errorf("Error when copying data %s (%T: %v)", d.URL, err, err)
   110  	}
   111  
   112  	logger.Info("Successfully downloaded \"%s\" %d bytes", d.Path, bytes)
   113  
   114  	return nil
   115  }
   116  
   117  type downloadError struct {
   118  	s string
   119  }
   120  
   121  func (e *downloadError) Error() string {
   122  	return e.s
   123  }