go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/scheduler/appengine/messages/config.proto

// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package scheduler.config;

option go_package = "go.chromium.org/luci/scheduler/appengine/messages";

import "go.chromium.org/luci/common/proto/options.proto";
import "google/protobuf/duration.proto";

option (luci.file_metadata) = {
  doc_url: "https://config.luci.app/schemas/projects:luci-scheduler.cfg";
};


// ProjectConfig defines a schema for a config file that describe jobs belonging
// to some project.
message ProjectConfig {
  reserved 4;
  reserved "security_options";

  // Job is a set of jobs defined in the project.
  repeated Job job = 1;
  // Trigger is a set of triggering jobs defined in the project.
  repeated Trigger trigger = 2;
  // Deprecated and unused. Use realms permissions instead.
  repeated AclSet acl_sets = 3 [deprecated = true];
}


// Deprecated in favor of LUCI Realms. This proto is totally unused now, exists
// only to not break older configs that still may have deprecated fields
// populated.
message Acl {
  enum Role {
    READER = 0;
    TRIGGERER = 2;
    OWNER = 1;
  }
  Role role = 1 [deprecated = true];
  string granted_to = 2 [deprecated = true];
}


// Deprecated in favor of LUCI Realms. This proto is totally unused now, exists
// only to not break older configs that still may have deprecated fields
// populated.
message AclSet {
  string name = 1 [deprecated = true];
  repeated Acl acls = 2 [deprecated = true];
}


// TriggeringPolicy defines a function that decides when and how to launch a
// job invocation, given the job's current state and a set of pending triggers.
message TriggeringPolicy {
  enum Kind {
    // A placeholder for unrecognized policy kind.
    UNDEFINED = 0;

    // A greedy triggering function that takes all pending triggers (up to
    // max_batch_size limit) and collapses them into one new invocation,
    // deriving its properties from the most recent trigger alone. It doesn't
    // wait for a full batch, nor tries to batch evenly.
    GREEDY_BATCHING = 1;

    // A logarithmic triggering function that takes floor(log(base,N)) pending
    // triggers (at least 1 and up to max_batch_size limit) and collapses them
    // into one new invocation, deriving its properties from the most recent
    // trigger alone, where N is the total number of pending triggers and k is
    // specified by the log_base field below.
    LOGARITHMIC_BATCHING = 2;

    // A triggering function that prioritizes the most recent pending triggers.
    // Triggers stay pending until they either become the most recent pending
    // trigger or expire. The timeout for pending triggers is specified by the
    // pending_timeout field below.
    NEWEST_FIRST = 3;
  }

  // Defines an algorithm to use for the triggering decisions.
  //
  // See comments for Kind enum field.
  //
  // Default is GREEDY_BATCHING.
  Kind kind = 1;

  // Limits number of job invocations running at the same time.
  //
  // If the number of current active invocations is more or equal to this
  // setting, the triggering function will be skipped completely, since it isn't
  // allowed to trigger anything anyway.
  //
  // Default is 1.
  int64 max_concurrent_invocations = 2;

  // Limits how many triggers can be put into one invocation request.
  //
  // For example, setting this to 1 will make each trigger launch its own
  // invocation. This value is ignore dby NEWEST_FIRST, since batching
  // isn't well-defined in that policy kind.
  //
  // Default is 1000 (which is ~= unlimited).
  int64 max_batch_size = 3;

  // Base of the logarithm operation during logarithmic batching.
  //
  // For example, setting this to 2, will cause 3 out of 8 pending triggers to
  // be combined into a single invocation with LOGARITHMIC_BATCHING kind. This
  // value is ignored by other policy kinds. Must be larger or equal to 1.0001
  // for numerical stability reasons.
  //
  // Required.
  float log_base = 4;

  // How long until a pending trigger is discarded.
  //
  // For example, setting this to 1 day will cause triggers that stay pending
  // for at least 1 day to be removed from consideration. This value is ignored
  // by policy kinds other than NEWEST_FIRST, which can starve old triggers and
  // cause the pending triggers list to grow without bound.
  //
  // Default is 7 days.
  google.protobuf.Duration pending_timeout = 5;
}


// Job specifies a single regular job belonging to a project.
//
// Such jobs runs on a schedule or can be triggered by some trigger.
message Job {
  reserved 4;
  reserved 102;

  // Id is a name of the job (unique for the project).
  //
  // Must match '^[0-9A-Za-z_\-\. \)\(]{1,100}$'.
  string id = 1;

  // Realm is a name of a LUCI realm within the project to use for ACLs.
  //
  // Must match `^[a-z0-9_\.\-/]{1,400}$` or be literals "@root" or "@legacy".
  // If not set, defaults to "@legacy".
  //
  // ACLs associated with a realm are defined separately in realms.cfg file.
  // That way they can be shared by multiple services.
  string realm = 8;

  // Schedule describes when to run the job.
  //
  // A job with a schedule can still be triggered by other triggering jobs
  // and via "Trigger" button in UI.
  //
  // Supported kinds of schedules (illustrated by examples):
  //   - "* 0 * * * *": cron-like expression, in a syntax supported by
  //     https://github.com/gorhill/cronexpr (see its docs for full reference).
  //     The cron engine will attempt to start a job at specified moments in
  //     time (based on UTC clock). If when triggering a job, previous
  //     invocation is still running, an overrun will be recorded (and next
  //     attempt to start a job happens based on the schedule, not when the
  //     previous invocation finishes). Some examples:
  //       "0 */3 * * * *" - each 3 hours: at 12:00 AM UTC, 3:00 AM UTC, ...
  //       "0 */3 * * *" - exact same thing (last field is optional)
  //       "0 1/3 * * *" - each 3 hours but starting 1:00 AM UTC
  //       "0 2,10,18 * * *" - at 2 AM UTC, 10 AM UTC, 6 PM UTC
  //       "0 7 * * *" - at 7 AM UTC, once a day.
  //   - "with 10s interval": runs invocations in a loop, waiting 10s after
  //     finishing invocation before starting a new one. Overruns are not
  //     possible.
  //   - "continuously" is alias for "with 0s interval", meaning the job will
  //     run in a loop without any pauses.
  //   - "triggered" schedule indicates that job is only started via a trigger.
  //
  // Default is "triggered".
  string schedule = 2;

  // Disabled is true to disable this job.
  //
  // Disabled job is equivalent to a deleted job: it can't be triggered, it
  // can't be referenced by other jobs and it doesn't show up in UI or API.
  //
  // Use this instead of commenting out the definition in case you want to
  // temporarily git rid of the job.
  bool disabled = 3;

  // Deprecated and unused. Use realms permissions instead.
  repeated Acl acls = 5 [deprecated = true];
  // Deprecated and unused. Use realms permissions instead.
  repeated string acl_sets = 6 [deprecated = true];

  // TriggeringPolicy defines how job handles incoming triggering events.
  //
  // If not specified defaults to GREEDY_BATCHING with 1 max concurrent
  // invocation. See comments in TriggeringPolicy for more details.
  TriggeringPolicy triggering_policy = 7;

  // One and only one field below must be set. It defines what this job does.

  // Noop is used for testing. It is "do nothing" task.
  NoopTask noop = 100;
  // UrlFetch can be used to make a simple HTTP call.
  UrlFetchTask url_fetch = 101;
  // BuildbucketTask can be used to schedule buildbucket job.
  BuildbucketTask buildbucket = 103;
}


// Trigger specifies a job that triggers other jobs.
//
// It is a special kind of a job that periodically checks the state of the world
// and triggers other jobs.
message Trigger {
  // Id is a name of the job (unique for the project).
  //
  // Must match '^[0-9A-Za-z_\-\. \)\(]{1,100}$'. It's in the same namespace as
  // regular jobs.
  string id = 1;

  // Realm is a name of a LUCI realm within the project to use for ACLs.
  //
  // Must match `^[a-z0-9_\.\-/]{1,400}$` or be literals "@root" or "@legacy".
  // If not set, defaults to "@legacy".
  //
  // ACLs associated with a realm are defined separately in realms.cfg file.
  // That way they can be shared by multiple services.
  string realm = 7;

  // Schedule describes when to run this triggering job.
  //
  // See Job.schedule for more info. Default is "with 30s interval".
  string schedule = 2;

  // Disabled is true to disable this job.
  //
  // Se Job.disabled for more info.
  bool disabled = 3;

  // Deprecated and unused. Use realms permissions instead.
  repeated Acl acls = 4 [deprecated = true];
  // Deprecated and unused. Use realms permissions instead.
  repeated string acl_sets = 5 [deprecated = true];

  // TriggeringPolicy defines how job handles incoming triggering events.
  //
  // It is rare for a trigger itself to have a non-default triggering policy,
  // so most likely you should not touch this field.
  TriggeringPolicy triggering_policy = 6;

  // Triggers are IDs of jobs triggered by this trigger.
  repeated string triggers = 200;

  // One and only one field below must be set. It defines what this trigger
  // polls.

  // Noop is used for testing. It is "do nothing" trigger.
  NoopTask noop = 100;
  // Gitiles is used to trigger jobs for new commits on Gitiles.
  GitilesTask gitiles = 101;
}


// NoopTask is used for testing. It is a "do nothing" task that can emit fake
// triggers.
message NoopTask {
  int64 sleep_ms = 1;
  int64 triggers_count = 2;
}


// GitilesTask specifies parameters of what repo and which refs to watch for new
// commits.
//
// GitilesTask will trigger other jobs if either:
//   * ref's tip has changed (e.g. new commit landed on a ref),
//   * a ref has just been created from Scheduler point of view, which is
//     treated as a single new commit.
//     For example, if you configure new GitilesTask or add a ref not previously
//     watched to the existing GitilesTask, then Scheduler will "discover" this
//     ref as newly created and emit exactly 1 trigger for the ref's current
//     tip.
//
//
// LIMITATIONS:
//
//  1. Per each fast-forward ref's tip change as observed by scheduler,
//     no more than 50 latest (according to gitiles default ordering) commits
//     will result in emitted triggers.
//     This is a safeguard against mistaken or deliberate but unusual git push
//     actions, which typically don't have intent of triggering a build for each
//     such commit.
//     If you absolutely need to act on every new commit even in such cases,
//     you will need to roll your own implementation, though you may still find
//     Scheduler GitilesTask useful to trigger your code on new changes.
//
//  2. No hard latency guarantee. A single invocation of GitilesTask may not be
//     able to process all the backlog. This is particularly true after a config
//     change changes the set of watched ref. However, GitilesTask is designed
//     to make continuous progress with each invocation.
message GitilesTask {
  // Repo is the URL of the Gitiles repository.
  string repo = 1;

  // Refs is a list of Git references to track.
  //
  // Each ref can be either:
  //   * a fully qualified ref like "refs/heads/master" or "refs/tags/v1.2.3"
  //   * a regular expression with "regexp:" prefix to match multiple refs, e.g.
  //     "regexp:refs/heads/[^/]+" or "regexp:refs/branch-heads/\d+\.\d+", but
  //     the regular expression should have a literal prefix with at least two
  //     slashes present, e.g. "refs/release-\d+/foobar" is not allowed, because
  //     the literal prefix "refs/release-" contains only one slash. The regexp
  //     should not start with ^ or end with $ as they will be added
  //     automatically.
  //
  // Each tracked ref, either fully qualified or regexp, must match at least 1
  // ref in gitiles output.
  repeated string refs = 2;

  // Optional path_regexps and path_regexps_exclude are lists of regular
  // expressions limiting emitted triggers only to commits whose tree diff
  // touches relevant files.
  //
  // If neither path_regexps nor path_regexps_exclude is specified (default),
  // emits triggers for any commit.
  // Specifying just path_regexps_exclude is not allowed.
  //
  // path_regexps and path_regexps_exclude rules apply on each individual commit
  // independently. For each commit, a set of all filepaths referenced in Git
  // commit diff is used for matching.  On top of simple file additions and file
  // modifications, this also includes: file removals, file moves (old and new
  // path is considered), and changing of file metadata (e.g., chmod +x). This
  // doesn't include directories (git doesn't track them explicitly).
  //
  // Triggers are emitted for a commit if only if at least 1 touched filepath
  //  1. is not matched by any path_regexps_exclude,
  //  2. AND is matched at least 1 path_regexps
  // **subject to caveats below** for exceptional cases.
  //
  // Each path_regexps and path_regexps_exclude is a regular expression
  // "a/b/c/.+". It should not start with ^ or end with $ as they will be added
  // automatically.
  //
  // CAVEATS:
  //  1. path_regexps: ".+"
  //     will NOT match commits which modify no files (aka empty commits) and
  //     as such differs from default case of not specifying any `path_regexps`.
  //  2. Per GitilesTask doc, if a ref fast-forwards >=50 commits, only the last
  //     50 commits are checked. If none of them matches any path specified here,
  //     no trigger is emitted, even if there is an unexamined commit which
  //     matches the path.
  //     TODO(tandrii): it's possible to improve this by examining diff
  //     between ref's last and new tips, but one has to worry about gitiles
  //     taking potentially very long time to compute it.
  //
  repeated string path_regexps = 3;
  repeated string path_regexps_exclude = 4;
}


// UrlFetchTask specifies parameters for simple HTTP call.
message UrlFetchTask {
  // Method is HTTP method to use, such as "GET" or "POST". Default is "GET".
  string method = 1;
  // Url to send the request to.
  string url = 2;
  // Timeout is how long to wait for request to complete. Default is 60 sec.
  int32 timeout_sec = 3;
  // TODO: add more.
}


// BuildbucketTask specifies parameters of Buildbucket-based jobs.
message BuildbucketTask {
  // Server is hostname of the buildbucket service to use.
  //
  // Typically, "cr-buildbucket.appspot.com".
  string server = 1;

  // Bucket defines what bucket to add the task to.
  //
  // Supported formats are:
  //    * "<bucket>" - for a bucket in the current project.
  //    * "<project>:<bucket>" - for a bucket in another project.
  //    * "luci.<project>.<bucket>" - legacy v1 bucket name.
  //
  // If empty, defaults to `realm` field in the parent Job message.
  string bucket = 2;

  // Builder defines what builder within the bucket to launch.
  string builder = 3;

  // Properties is arbitrary "key:value" pairs describing the task.
  // TODO(tandrii): which properties will be overridden if triggered?
  repeated string properties = 4;

  // Tags is a list of tags (as "key:value" pairs) to assign to the task.
  repeated string tags = 5;
}


////////////////////////////////////////////////////////////////////////////////
// Internal stuff.

// TaskDefWrapper is a union type of all possible tasks known to the scheduler.
//
// It is used internally when storing jobs in the datastore.
message TaskDefWrapper {
  reserved 3;

  NoopTask noop = 1;
  UrlFetchTask url_fetch = 2;
  BuildbucketTask buildbucket = 4;
  GitilesTask gitiles = 5;
}