github.com/pquerna/agent@v2.1.8+incompatible/agent/artifact_downloader.go (about) 1 package agent 2 3 import ( 4 "os" 5 "path/filepath" 6 "strings" 7 8 "github.com/buildkite/agent/api" 9 "github.com/buildkite/agent/logger" 10 "github.com/buildkite/agent/pool" 11 ) 12 13 type ArtifactDownloader struct { 14 // The APIClient that will be used when uploading jobs 15 APIClient *api.Client 16 17 // The ID of the Build 18 BuildID string 19 20 // The query used to find the artifacts 21 Query string 22 23 // Which step should we look at for the jobs 24 Step string 25 26 // Where we'll be downloading artifacts to 27 Destination string 28 } 29 30 func (a *ArtifactDownloader) Download() error { 31 // Turn the download destination into an absolute path and confirm it exists 32 downloadDestination, _ := filepath.Abs(a.Destination) 33 fileInfo, err := os.Stat(downloadDestination) 34 if err != nil { 35 logger.Fatal("Could not find information about destination: %s", downloadDestination) 36 } 37 if !fileInfo.IsDir() { 38 logger.Fatal("%s is not a directory", downloadDestination) 39 } 40 41 // Find the artifacts that we want to download 42 searcher := ArtifactSearcher{BuildID: a.BuildID, APIClient: a.APIClient} 43 artifacts, err := searcher.Search(a.Query, a.Step) 44 if err != nil { 45 return err 46 } 47 48 artifactCount := len(artifacts) 49 50 if artifactCount == 0 { 51 logger.Info("No artifacts found for downloading") 52 } else { 53 logger.Info("Found %d artifacts. Starting to download to: %s", artifactCount, downloadDestination) 54 55 p := pool.New(pool.MaxConcurrencyLimit) 56 errors := []error{} 57 58 for _, artifact := range artifacts { 59 // Create new instance of the artifact for the goroutine 60 // See: http://golang.org/doc/effective_go.html#channels 61 artifact := artifact 62 63 p.Spawn(func() { 64 var err error 65 66 // Handle downloading from S3 67 if strings.HasPrefix(artifact.UploadDestination, "s3://") { 68 err = S3Downloader{ 69 Path: artifact.Path, 70 Bucket: artifact.UploadDestination, 71 Destination: downloadDestination, 72 Retries: 5, 73 DebugHTTP: a.APIClient.DebugHTTP, 74 }.Start() 75 } else { 76 err = Download{ 77 URL: artifact.URL, 78 Path: artifact.Path, 79 Destination: downloadDestination, 80 Retries: 5, 81 DebugHTTP: a.APIClient.DebugHTTP, 82 }.Start() 83 } 84 85 // If the downloaded encountered an error, lock 86 // the pool, collect it, then unlock the pool 87 // again. 88 if err != nil { 89 logger.Error("Failed to download artifact: %s", err) 90 91 p.Lock() 92 errors = append(errors, err) 93 p.Unlock() 94 } 95 }) 96 } 97 98 p.Wait() 99 100 if len(errors) > 0 { 101 logger.Fatal("There were errors with downloading some of the artifacts") 102 } 103 } 104 105 return nil 106 }