go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/cv/internal/tryjob/execute/reuse_backend.go (about)

     1  // Copyright 2022 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package execute
    16  
    17  import (
    18  	"context"
    19  
    20  	"go.chromium.org/luci/common/clock"
    21  	"go.chromium.org/luci/common/errors"
    22  	"go.chromium.org/luci/common/logging"
    23  	"go.chromium.org/luci/common/retry/transient"
    24  	"go.chromium.org/luci/gae/service/datastore"
    25  
    26  	"go.chromium.org/luci/cv/internal/tryjob"
    27  )
    28  
    29  // findReuseInBackend finds reusable Tryjobs by querying the backend (e.g.
    30  // Buildbucket).
    31  func (w *worker) findReuseInBackend(ctx context.Context, definitions []*tryjob.Definition) (map[*tryjob.Definition]*tryjob.Tryjob, error) {
    32  	candidates := make(map[*tryjob.Definition]*tryjob.Tryjob, len(definitions))
    33  	cutOffTime := clock.Now(ctx).Add(-staleTryjobAge)
    34  	err := w.backend.Search(ctx, w.cls, definitions, w.run.ID.LUCIProject(), func(tj *tryjob.Tryjob) bool {
    35  		// backend.Search returns matching Tryjob from newest to oldest, if backend
    36  		// starts to return stale Tryjob (Tryjob created before cutoff time), then
    37  		// there's no point resuming the search as none of the returning Tryjobs
    38  		// will be reusable anyway.
    39  		if createTime := tj.Result.GetCreateTime(); createTime != nil && createTime.AsTime().Before(cutOffTime) {
    40  			return false
    41  		}
    42  
    43  		switch candidate, ok := candidates[tj.Definition]; {
    44  		case ok:
    45  			// Matching Tryjob already found.
    46  			if tj.Result.GetCreateTime().AsTime().After(candidate.Result.GetCreateTime().AsTime()) {
    47  				logging.Errorf(ctx, "FIXME(crbug/1369200): backend.Search is expected to return tryjob from newest to oldest; However, got %s before %s.", candidate.Result, tj.Result)
    48  			}
    49  		case w.knownExternalIDs.Has(string(tj.ExternalID)):
    50  		case canReuseTryjob(ctx, tj, w.run.Mode) == reuseDenied:
    51  		default:
    52  			candidates[tj.Definition] = tj
    53  		}
    54  		return len(candidates) < len(definitions)
    55  	})
    56  	switch {
    57  	case err != nil:
    58  		return nil, err
    59  	case len(candidates) == 0:
    60  		return nil, nil
    61  	}
    62  
    63  	var innerErr error
    64  	var tryjobs []*tryjob.Tryjob
    65  	err = datastore.RunInTransaction(ctx, func(ctx context.Context) (err error) {
    66  		defer func() { innerErr = err }()
    67  		eids := make([]tryjob.ExternalID, 0, len(candidates))
    68  		definitions = make([]*tryjob.Definition, 0, len(candidates))
    69  		for def, tj := range candidates {
    70  			eids = append(eids, tj.ExternalID)
    71  			definitions = append(definitions, def)
    72  		}
    73  		tryjobs, err = tryjob.ResolveToTryjobs(ctx, eids...)
    74  		if err != nil {
    75  			return err
    76  		}
    77  		for i, tj := range tryjobs {
    78  			if tj == nil {
    79  				tj = w.makeBaseTryjob(ctx)
    80  				tryjobs[i] = tj
    81  			} else {
    82  				// The Tryjob already in datastore. This shouldn't normally
    83  				// happen but be defensive here when it actually happens.
    84  				tj.EVersion++
    85  				tj.EntityUpdateTime = clock.Now(ctx).UTC()
    86  				logging.Warningf(ctx, "tryjob %q was found reusable in backend, but "+
    87  					"it already has a corresponding Tryjob entity (ID: %d). Ideally, "+
    88  					"this Tryjob should be surfaced at the first attempt to search for "+
    89  					"reusable Tryjob in datastore", tj.ExternalID, tj.ID)
    90  			}
    91  			tj.Definition = definitions[i]
    92  			candidate := candidates[tj.Definition]
    93  			tj.ExternalID = candidate.ExternalID
    94  			tj.Status = candidate.Status
    95  			tj.Result = candidate.Result
    96  			if runID := w.run.ID; tj.AllWatchingRuns().Index(runID) < 0 {
    97  				tj.ReusedBy = append(tj.ReusedBy, runID)
    98  			}
    99  			candidates[tj.Definition] = tj
   100  		}
   101  		return tryjob.SaveTryjobs(ctx, tryjobs, w.rm.NotifyTryjobsUpdated)
   102  	}, nil)
   103  	switch {
   104  	case innerErr != nil:
   105  		return nil, innerErr
   106  	case err != nil:
   107  		return nil, errors.Annotate(err, "failed to commit transaction").Tag(transient.Tag).Err()
   108  	}
   109  
   110  	return candidates, nil
   111  }