go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/cv/internal/tryjob/model.go

// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tryjob

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/gae/service/datastore"

	"go.chromium.org/luci/cv/internal/common"
)

const (
	// TryjobKind is the Datastore entity kind for Tryjob.
	TryjobKind = "Tryjob"
)

// Tryjob is an entity tracking CV Tryjobs.
type Tryjob struct {
	// $kind must match TryjobKind.
	_kind  string                `gae:"$kind,Tryjob"`
	_extra datastore.PropertyMap `gae:"-,extra"`

	// ID is the Tryjob ID, autogenerated by the Datastore.
	ID common.TryjobID `gae:"$id"`
	// ExternalID is a Tryjob ID in external system, e.g. Buildbucket.
	//
	// There can be at most one Tryjob with a given ExternalID.
	ExternalID ExternalID `gae:",noindex"` // Indexed in tryjobMap entities.
	// EVersion is the entity version.
	//
	// It increments by one upon every successful modification.
	EVersion int64 `gae:",noindex"`
	// EntityCreateTime is the timestamp when this entity was created.
	//
	// NOTE: This is not the backend's tryjob creation time, which is stored in
	// .Result.CreateTime.
	EntityCreateTime time.Time `gae:",noindex"`
	// UpdateTime is the timestamp when this entity was last updated.
	//
	// NOTE: This is not the backend's tryjob update time, which is stored in
	// .Result.UpdateTime.
	EntityUpdateTime time.Time `gae:",noindex"`

	// RetentionKey is for data retention purpose.
	//
	// It is indexed and tries to avoid hot areas in the index. The format is
	// `{shard_key}/{unix_time_of_EntityUpdateTime}`. Shard key is the last 2
	// digit of ID with left padded zero. Unix timestamp is a 10 digit integer
	// with left padded zero if necessary.
	RetentionKey string

	// ReuseKey is used to quickly decide if this Tryjob can be reused by a run.
	//
	// Note that, even if reuse is allowed here, reuse is still subjected to
	// other restrictions (for example, Tryjob is not fresh enough for the run).
	//
	// reusekey is currently computed in the following way:
	//  base64(
	//    sha256(
	//      '\0'.join(sorted('%d/%d' % (cl.ID, cl.minEquiPatchSet) for cl in cls))
	//    )
	//  )
	//
	// Indexed
	ReuseKey string

	// Definition of the tryjob.
	//
	// Immutable.
	Definition *Definition

	// Status of the Tryjob.
	Status Status `gae:",noindex"`

	// Result of the Tryjob.
	//
	// Must be set if Status is ENDED.
	// May be set if Status is TRIGGERED.
	//
	// It's used by the Run Manager.
	Result *Result

	// LaunchedBy is the Run that launches this Tryjob.
	//
	// May be unset if the Tryjob was not launched by CV (e.g. through Gerrit
	// UI), in which case ReusedBy should have at least one Run.
	LaunchedBy common.RunID `gae:",noindex"`

	// ReusedBy are the Runs that are interested in the result of this Tryjob.
	ReusedBy common.RunIDs `gae:",noindex"`

	// CLPatchsets is an array of CLPatchset that each identify a specific
	// patchset in a specific CL.
	//
	// The values are to be computed by MakeCLPatchset().
	// See its documentation for details.
	//
	// Sorted and Indexed.
	CLPatchsets CLPatchsets

	// UntriggeredReason is the reason why LUCI CV doesn't trigger the Tryjob.
	UntriggeredReason string `gae:",noindex"`
}

// DO NOT decrease the shard. It will cause olds tryjobs that are out of
// retention period in the shard not getting wiped out.
const retentionKeyShards = 100

var _ datastore.PropertyLoadSaver = (*Tryjob)(nil)

// Save implements datastore.PropertyLoadSaver.
//
// Makes sure the EntityUpdateTime and RetentionKey are always updated.
func (tj *Tryjob) Save(withMeta bool) (datastore.PropertyMap, error) {
	if tj.EntityUpdateTime.IsZero() { // be defensive
		tj.EntityUpdateTime = datastore.RoundTime(time.Now().UTC())
	}
	tj.RetentionKey = fmt.Sprintf("%02d/%010d", tj.ID%retentionKeyShards, tj.EntityUpdateTime.Unix())
	return datastore.GetPLS(tj).Save(withMeta)
}

// Load implements datastore.PropertyLoadSaver.
func (tj *Tryjob) Load(p datastore.PropertyMap) error {
	return datastore.GetPLS(tj).Load(p)
}

// CondDelete conditionally deletes Tryjob and corresponding tryjobMap entities.
//
// The deletion would only proceed if the loaded tryjob has the same EVersion
// as the provided one. The deletion would happen in a transaction to make sure
// the deletion of Tryjob and tryjobMap entities are atomic.
func CondDelete(ctx context.Context, tjID common.TryjobID, expectedEVersion int64) error {
	if expectedEVersion <= 0 {
		return errors.New("expected EVersion must be larger than 0")
	}

	return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		tj := &Tryjob{ID: tjID}
		switch err := datastore.Get(ctx, tj); {
		case errors.Is(err, datastore.ErrNoSuchEntity):
			return nil // tryjob already gets deleted
		case err != nil:
			return errors.Annotate(err, "failed to load tryjob %d", tjID).Tag(transient.Tag).Err()
		case tj.EVersion != expectedEVersion:
			return errors.Reason("request to delete tryjob %d at EVersion: %d, got EVersion: %d", tjID, expectedEVersion, tj.EVersion).Err()
		}
		toDelete := []any{tj}
		if tj.ExternalID != "" {
			// some tryjobs might not have external ID populated.
			toDelete = append(toDelete, &tryjobMap{ExternalID: tj.ExternalID})
		}
		if err := datastore.Delete(ctx, toDelete); err != nil {
			return errors.Annotate(err, "failed to delete tryjob %d", tjID).Tag(transient.Tag).Err()
		}
		return nil
	}, nil)
}

// tryjobMap is intended to quickly determine if a given ExternalID is
// associated with a Tryjob entity in the datastore.
//
// This also ensures that at most one TryjobID will be associated with a given
// ExternalID.
type tryjobMap struct {
	_kind string `gae:"$kind,TryjobMap"`

	// ExternalID is an ID for the tryjob in the external backend.
	//
	// Making this the key of the map ensures uniqueness.
	ExternalID ExternalID `gae:"$id"`

	// InternalID is auto-generated by Datastore for Tryjob entity.
	InternalID common.TryjobID `gae:",noindex"` // int64. Indexed in Tryjob entities.
}

// LUCIProject() returns the project in the context of which the Tryjob is
// updated, and which is thus allowed to "read" the Tryjob.
//
// In the case of Buildbucket, this may be different from the LUCI project to
// which the corresponding build belongs. For example, consider a "v8" project
// with configuration saying to trigger "chromium/try/linux_rel" builder: when
// CV triggers a new tryjob T for a "v8" Run, T.LUCIProject() will be "v8" even
// though the build itself will be in the "chromium/try" Buildbucket bucket.
//
// In general, a Run of project P must not re-use tryjob T if
// T.LUCIProject() != P, until it has been verified with the tryjob backend
// that P has access to T.
func (t *Tryjob) LUCIProject() string {
	if t.LaunchedBy != "" {
		return t.LaunchedBy.LUCIProject()
	}
	if len(t.ReusedBy) == 0 {
		panic("tryjob is not associated with any runs")
	}
	return t.ReusedBy[0].LUCIProject()
}

// AllWatchingRuns returns the IDs for the Runs that care about this tryjob.
//
// This includes the triggerer (if the tryjob was triggered by CV) and all the
// Runs reusing this tryjob (if any).
func (t *Tryjob) AllWatchingRuns() common.RunIDs {
	ret := make(common.RunIDs, 0, 1+len(t.ReusedBy))
	if t.LaunchedBy != "" {
		ret = append(ret, t.LaunchedBy)
	}
	return append(ret, t.ReusedBy...)
}

// IsEnded checks whether the Tryjob's status is final (can not change again).
func (t *Tryjob) IsEnded() bool {
	switch t.Status {
	case Status_CANCELLED, Status_ENDED, Status_UNTRIGGERED:
		return true
	case Status_PENDING, Status_TRIGGERED:
		return false
	default:
		panic(fmt.Errorf("unexpected tryjob status %s", t.Status.String()))
	}
}

// CLPatchsets is a slice of `CLPatchset`s.
//
// Implements sort.Interface
type CLPatchsets []CLPatchset

// Len implements sort.Interface.
func (c CLPatchsets) Len() int {
	return len(c)
}

// Less implements sort.Interface.
func (c CLPatchsets) Less(i int, j int) bool {
	return c[i] < c[j]
}

// Swap implements sort.Interface.
func (c CLPatchsets) Swap(i int, j int) {
	c[i], c[j] = c[j], c[i]
}

// CLPatchset is a value computed combining a CL's ID and a patchset number.
//
// This is intended to efficiently query Tryjob entities associated with a
// patchset.
//
// The values are hex string encoded and padded so that lexicographical sorting
// will put the patchsets for a given CL together.
type CLPatchset string

const clPatchsetEncodingVersion = 1

// MakeCLPatchset computes a new CLPatchset value.
func MakeCLPatchset(cl common.CLID, patchset int32) CLPatchset {
	return CLPatchset(fmt.Sprintf("%02x/%016x/%08x", clPatchsetEncodingVersion, cl, patchset))
}

// Parse extracts CLID and Patchset number from a valid CLPatchset value.
//
// Returns an error if the format is unexpected.
func (cp CLPatchset) Parse() (common.CLID, int32, error) {
	var clid, patchset int64
	values := strings.Split(string(cp), "/")
	// If any valid encoding versions require a different number of values,
	// check it here.
	switch len(values) {
	case 3:
		// Version 1 requires three slash-separated values.
	default:
		return 0, 0, errors.Reason("CLPatchset in unexpected format %q", cp).Err()
	}

	ver, err := strconv.ParseInt(values[0], 16, 32)
	switch {
	case err != nil:
		return 0, 0, errors.Annotate(err, "version segment in unexpected format %q", values[0]).Err()
	case ver == clPatchsetEncodingVersion:
		if len(values) != 3 {
			panic(fmt.Errorf("impossible: number of values is not 3"))
		}
		clid, err = strconv.ParseInt(values[1], 16, 64)
		if err != nil {
			return 0, 0, errors.Annotate(err, "clid segment in unexpected format %q", values[1]).Err()
		}
		patchset, err = strconv.ParseInt(values[2], 16, 32)
		if err != nil {
			return 0, 0, errors.Annotate(err, "patchset segment in unexpected format %q", values[2]).Err()
		}
		return common.CLID(clid), int32(patchset), nil
	default:
		return 0, 0, errors.Reason("unsupported version %d", ver).Err()
	}
}