github.com/stevenmatthewt/agent@v3.5.4+incompatible/agent/artifact_batch_creator.go (about) 1 package agent 2 3 import ( 4 "time" 5 6 "github.com/buildkite/agent/api" 7 "github.com/buildkite/agent/logger" 8 "github.com/buildkite/agent/retry" 9 ) 10 11 type ArtifactBatchCreator struct { 12 // The APIClient that will be used when uploading jobs 13 APIClient *api.Client 14 15 // The ID of the Job that these artifacts belong to 16 JobID string 17 18 // All the artifacts that need to be created 19 Artifacts []*api.Artifact 20 21 // Where the artifacts are being uploaded to on the command line 22 UploadDestination string 23 } 24 25 func (a *ArtifactBatchCreator) Create() ([]*api.Artifact, error) { 26 length := len(a.Artifacts) 27 chunks := 30 28 29 // Split into the artifacts into chunks so we're not uploading a ton of 30 // files at once. 31 for i := 0; i < length; i += chunks { 32 j := i + chunks 33 if length < j { 34 j = length 35 } 36 37 // The artifacts that will be uploaded in this chunk 38 theseArtiacts := a.Artifacts[i:j] 39 40 // An ID is required so Buildkite can ensure this create 41 // operation is idompotent (if we try and upload the same ID 42 // twice, it'll just return the previous data and skip the 43 // upload) 44 batch := &api.ArtifactBatch{api.NewUUID(), theseArtiacts, a.UploadDestination} 45 46 logger.Info("Creating (%d-%d)/%d artifacts", i, j, length) 47 48 var creation *api.ArtifactBatchCreateResponse 49 var resp *api.Response 50 var err error 51 52 // Retry the batch upload a couple of times 53 err = retry.Do(func(s *retry.Stats) error { 54 creation, resp, err = a.APIClient.Artifacts.Create(a.JobID, batch) 55 if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 404 || resp.StatusCode == 500) { 56 s.Break() 57 } 58 if err != nil { 59 logger.Warn("%s (%s)", err, s) 60 } 61 62 return err 63 }, &retry.Config{Maximum: 10, Interval: 5 * time.Second}) 64 65 // Did the batch creation eventually fail? 66 if err != nil { 67 return nil, err 68 } 69 70 // Save the id and instructions to each artifact 71 index := 0 72 for _, id := range creation.ArtifactIDs { 73 theseArtiacts[index].ID = id 74 theseArtiacts[index].UploadInstructions = creation.UploadInstructions 75 index += 1 76 } 77 } 78 79 return a.Artifacts, nil 80 }