github.com/pquerna/agent@v2.1.8+incompatible/agent/artifact_batch_creator.go (about)

     1  package agent
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/buildkite/agent/api"
     7  	"github.com/buildkite/agent/logger"
     8  	"github.com/buildkite/agent/retry"
     9  )
    10  
    11  type ArtifactBatchCreator struct {
    12  	// The APIClient that will be used when uploading jobs
    13  	APIClient *api.Client
    14  
    15  	// The ID of the Job that these artifacts belong to
    16  	JobID string
    17  
    18  	// All the artifacts that need to be created
    19  	Artifacts []*api.Artifact
    20  
    21  	// Where the artifacts are being uploaded to on the command line
    22  	UploadDestination string
    23  }
    24  
    25  func (a *ArtifactBatchCreator) Create() ([]*api.Artifact, error) {
    26  	length := len(a.Artifacts)
    27  	chunks := 30
    28  
    29  	// Split into the artifacts into chunks so we're not uploading a ton of
    30  	// files at once.
    31  	for i := 0; i < length; i += chunks {
    32  		j := i + chunks
    33  		if length < j {
    34  			j = length
    35  		}
    36  
    37  		// The artifacts that will be uploaded in this chunk
    38  		theseArtiacts := a.Artifacts[i:j]
    39  
    40  		// An ID is required so Buildkite can ensure this create
    41  		// operation is idompotent (if we try and upload the same ID
    42  		// twice, it'll just return the previous data and skip the
    43  		// upload)
    44  		batch := &api.ArtifactBatch{api.NewUUID(), theseArtiacts, a.UploadDestination}
    45  
    46  		logger.Info("Creating (%d-%d)/%d artifacts", i, j, length)
    47  
    48  		var creation *api.ArtifactBatchCreateResponse
    49  		var resp *api.Response
    50  		var err error
    51  
    52  		// Retry the batch upload a couple of times
    53  		err = retry.Do(func(s *retry.Stats) error {
    54  			creation, resp, err = a.APIClient.Artifacts.Create(a.JobID, batch)
    55  			if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 404 || resp.StatusCode == 500) {
    56  				s.Break()
    57  			}
    58  			if err != nil {
    59  				logger.Warn("%s (%s)", err, s)
    60  			}
    61  
    62  			return err
    63  		}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})
    64  
    65  		// Did the batch creation eventually fail?
    66  		if err != nil {
    67  			return nil, err
    68  		}
    69  
    70  		// Save the id and instructions to each artifact
    71  		index := 0
    72  		for _, id := range creation.ArtifactIDs {
    73  			theseArtiacts[index].ID = id
    74  			theseArtiacts[index].UploadInstructions = creation.UploadInstructions
    75  			index += 1
    76  		}
    77  	}
    78  
    79  	return a.Artifacts, nil
    80  }