github.com/bshelton229/agent@v3.5.4+incompatible/agent/artifact_uploader.go (about) 1 package agent 2 3 import ( 4 "crypto/sha1" 5 "errors" 6 "fmt" 7 "io" 8 "os" 9 "path/filepath" 10 "runtime" 11 "strings" 12 "sync" 13 "time" 14 15 "github.com/buildkite/agent/api" 16 "github.com/buildkite/agent/logger" 17 "github.com/buildkite/agent/pool" 18 "github.com/buildkite/agent/retry" 19 zglob "github.com/mattn/go-zglob" 20 ) 21 22 const ( 23 ArtifactPathDelimiter = ";" 24 ) 25 26 type ArtifactUploader struct { 27 // The APIClient that will be used when uploading jobs 28 APIClient *api.Client 29 30 // The ID of the Job 31 JobID string 32 33 // The path of the uploads 34 Paths string 35 36 // Where we'll be uploading artifacts 37 Destination string 38 } 39 40 func (a *ArtifactUploader) Upload() error { 41 // Create artifact structs for all the files we need to upload 42 artifacts, err := a.Collect() 43 if err != nil { 44 return err 45 } 46 47 if len(artifacts) == 0 { 48 logger.Info("No files matched paths: %s", a.Paths) 49 } else { 50 logger.Info("Found %d files that match \"%s\"", len(artifacts), a.Paths) 51 52 err := a.upload(artifacts) 53 if err != nil { 54 return err 55 } 56 } 57 58 return nil 59 } 60 61 func isDir(path string) bool { 62 fi, err := os.Stat(path) 63 if err != nil { 64 return false 65 } 66 return fi.IsDir() 67 } 68 69 func (a *ArtifactUploader) Collect() (artifacts []*api.Artifact, err error) { 70 wd, err := os.Getwd() 71 if err != nil { 72 return nil, err 73 } 74 75 for _, globPath := range strings.Split(a.Paths, ArtifactPathDelimiter) { 76 globPath = strings.TrimSpace(globPath) 77 if globPath == "" { 78 continue 79 } 80 81 logger.Debug("Searching for %s", globPath) 82 83 // Resolve the globs (with * and ** in them), if it's a non-globbed path and doesn't exists 84 // then we will get the ErrNotExist that is handled below 85 files, err := zglob.Glob(globPath) 86 if err == os.ErrNotExist { 87 logger.Info("File not found: %s", globPath) 88 continue 89 } else if err != nil { 90 return nil, err 91 } 92 93 // Process each glob match into an api.Artifact 94 for _, file := range files { 95 absolutePath, err := filepath.Abs(file) 96 if err != nil { 97 return nil, err 98 } 99 100 // Ignore directories, we only want files 101 if isDir(absolutePath) { 102 logger.Debug("Skipping directory %s", file) 103 continue 104 } 105 106 // If a glob is absolute, we need to make it relative to the root so that 107 // it can be combined with the download destination to make a valid path. 108 // This is possibly weird and crazy, this logic dates back to 109 // https://github.com/buildkite/agent/commit/8ae46d975aa60d1ae0e2cc0bff7a43d3bf960935 110 // from 2014, so I'm replicating it here to avoid breaking things 111 if filepath.IsAbs(globPath) { 112 if runtime.GOOS == "windows" { 113 wd = filepath.VolumeName(absolutePath) + "/" 114 } else { 115 wd = "/" 116 } 117 } 118 119 path, err := filepath.Rel(wd, absolutePath) 120 if err != nil { 121 return nil, err 122 } 123 124 // Build an artifact object using the paths we have. 125 artifact, err := a.build(path, absolutePath, globPath) 126 if err != nil { 127 return nil, err 128 } 129 130 artifacts = append(artifacts, artifact) 131 } 132 } 133 134 return artifacts, nil 135 } 136 137 func (a *ArtifactUploader) build(path string, absolutePath string, globPath string) (*api.Artifact, error) { 138 // Temporarily open the file to get it's size 139 file, err := os.Open(absolutePath) 140 if err != nil { 141 return nil, err 142 } 143 defer file.Close() 144 145 // Grab it's file info (which includes it's file size) 146 fileInfo, err := file.Stat() 147 if err != nil { 148 return nil, err 149 } 150 151 // Generate a sha1 checksum for the file 152 hash := sha1.New() 153 io.Copy(hash, file) 154 checksum := fmt.Sprintf("%x", hash.Sum(nil)) 155 156 // Create our new artifact data structure 157 artifact := &api.Artifact{ 158 Path: path, 159 AbsolutePath: absolutePath, 160 GlobPath: globPath, 161 FileSize: fileInfo.Size(), 162 Sha1Sum: checksum, 163 } 164 165 return artifact, nil 166 } 167 168 func (a *ArtifactUploader) upload(artifacts []*api.Artifact) error { 169 var uploader Uploader 170 171 // Determine what uploader to use 172 if a.Destination != "" { 173 if strings.HasPrefix(a.Destination, "s3://") { 174 uploader = new(S3Uploader) 175 } else if strings.HasPrefix(a.Destination, "gs://") { 176 uploader = new(GSUploader) 177 } else { 178 return errors.New(fmt.Sprintf("Invalid upload destination: '%v'. Only s3:// and gs:// upload destinations are allowed. Did you forget to surround your artifact upload pattern in double quotes?", a.Destination)) 179 } 180 } else { 181 uploader = new(FormUploader) 182 } 183 184 // Setup the uploader 185 err := uploader.Setup(a.Destination, a.APIClient.DebugHTTP) 186 if err != nil { 187 return err 188 } 189 190 // Set the URL's of the artifacts based on the uploader 191 for _, artifact := range artifacts { 192 artifact.URL = uploader.URL(artifact) 193 } 194 195 // Create the artifacts on Buildkite 196 batchCreator := ArtifactBatchCreator{ 197 APIClient: a.APIClient, 198 JobID: a.JobID, 199 Artifacts: artifacts, 200 UploadDestination: a.Destination, 201 } 202 artifacts, err = batchCreator.Create() 203 if err != nil { 204 return err 205 } 206 207 // Prepare a concurrency pool to upload the artifacts 208 p := pool.New(pool.MaxConcurrencyLimit) 209 errors := []error{} 210 var errorsMutex sync.Mutex 211 212 // Create a wait group so we can make sure the uploader waits for all 213 // the artifact states to upload before finishing 214 var stateUploaderWaitGroup sync.WaitGroup 215 stateUploaderWaitGroup.Add(1) 216 217 // A map to keep track of artifact states and how many we've uploaded 218 artifactStates := make(map[string]string) 219 artifactStatesUploaded := 0 220 var artifactStatesMutex sync.Mutex 221 222 // Spin up a gourtine that'll uploading artifact statuses every few 223 // seconds in batches 224 go func() { 225 for artifactStatesUploaded < len(artifacts) { 226 statesToUpload := make(map[string]string) 227 228 // Grab all the states we need to upload, and remove 229 // them from the tracking map 230 // 231 // Since we mutate the artifactStates variable in 232 // multiple routines, we need to lock it to make sure 233 // nothing else is changing it at the same time. 234 artifactStatesMutex.Lock() 235 for id, state := range artifactStates { 236 statesToUpload[id] = state 237 delete(artifactStates, id) 238 } 239 artifactStatesMutex.Unlock() 240 241 if len(statesToUpload) > 0 { 242 artifactStatesUploaded += len(statesToUpload) 243 for id, state := range statesToUpload { 244 logger.Debug("Artifact `%s` has state `%s`", id, state) 245 } 246 247 // Update the states of the artifacts in bulk. 248 err = retry.Do(func(s *retry.Stats) error { 249 _, err = a.APIClient.Artifacts.Update(a.JobID, statesToUpload) 250 if err != nil { 251 logger.Warn("%s (%s)", err, s) 252 } 253 254 return err 255 }, &retry.Config{Maximum: 10, Interval: 5 * time.Second}) 256 257 if err != nil { 258 logger.Error("Error uploading artifact states: %s", err) 259 260 // Track the error that was raised. We need to 261 // aquire a lock since we mutate the errors 262 // slice in mutliple routines. 263 errorsMutex.Lock() 264 errors = append(errors, err) 265 errorsMutex.Unlock() 266 } 267 268 logger.Debug("Uploaded %d artfact states (%d/%d)", len(statesToUpload), artifactStatesUploaded, len(artifacts)) 269 } 270 271 // Check again for states to upload in a few seconds 272 time.Sleep(1 * time.Second) 273 } 274 275 stateUploaderWaitGroup.Done() 276 }() 277 278 for _, artifact := range artifacts { 279 // Create new instance of the artifact for the goroutine 280 // See: http://golang.org/doc/effective_go.html#channels 281 artifact := artifact 282 283 p.Spawn(func() { 284 // Show a nice message that we're starting to upload the file 285 logger.Info("Uploading artifact %s %s (%d bytes)", artifact.ID, artifact.Path, artifact.FileSize) 286 287 // Upload the artifact and then set the state depending 288 // on whether or not it passed. We'll retry the upload 289 // a couple of times before giving up. 290 err = retry.Do(func(s *retry.Stats) error { 291 err := uploader.Upload(artifact) 292 if err != nil { 293 logger.Warn("%s (%s)", err, s) 294 } 295 296 return err 297 }, &retry.Config{Maximum: 10, Interval: 5 * time.Second}) 298 299 var state string 300 301 // Did the upload eventually fail? 302 if err != nil { 303 logger.Error("Error uploading artifact \"%s\": %s", artifact.Path, err) 304 305 // Track the error that was raised. We need to 306 // aquire a lock since we mutate the errors 307 // slice in mutliple routines. 308 errorsMutex.Lock() 309 errors = append(errors, err) 310 errorsMutex.Unlock() 311 312 state = "error" 313 } else { 314 state = "finished" 315 } 316 317 // Since we mutate the artifactStates variable in 318 // multiple routines, we need to lock it to make sure 319 // nothing else is changing it at the same time. 320 artifactStatesMutex.Lock() 321 artifactStates[artifact.ID] = state 322 artifactStatesMutex.Unlock() 323 }) 324 } 325 326 // Wait for the pool to finish 327 p.Wait() 328 329 // Wait for the statuses to finish uploading 330 stateUploaderWaitGroup.Wait() 331 332 if len(errors) > 0 { 333 logger.Fatal("There were errors with uploading some of the artifacts") 334 } 335 336 return nil 337 }