go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/proto/config/project_config.proto

// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package luci.analysis.config;

import "google/protobuf/timestamp.proto";

import "go.chromium.org/luci/analysis/proto/config/test_variant_analysis_config.proto";

option go_package = "go.chromium.org/luci/analysis/proto/config;configpb";

// ProjectConfig is the project-specific configuration data for LUCI Analysis.
message ProjectConfig {
  // The project metadata (eg. display name).
  ProjectMetadata project_metadata = 6;

  // The last time this project configuration was updated.
  // LUCI Analysis sets and stores this value internally. Do not set
  // in your project's configuration file, it will be ignored.
  google.protobuf.Timestamp last_updated = 4;

  // Configuration for how to cluster test results.
  Clustering clustering = 5;

  // Configuration for automatic bug management.
  BugManagement bug_management = 9;

  // Configuration related to metrics in LUCI Analysis.
  Metrics metrics = 11;

  // Configuration for when tests are considered stable enough
  // to gate code changes. Only relevant for projects which integrate
  // with the TestVariants.QueryStability RPC to exonerate
  // unstable tests in presubmit.
  TestStabilityCriteria test_stability_criteria = 12;

  // Deprecated configuration follows. Setting these no longer
  // has any effect.

  // Per realm configurations.
  repeated RealmConfig realms = 3;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.default_bug_system instead.
  BugSystem bug_system = 7;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.monorail instead.
  MonorailProject monorail = 1;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.buganizer instead.
  BuganizerProject buganizer = 8;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.policies instead.
  ImpactThreshold bug_filing_threshold = 2;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.policies instead.
  repeated ImpactMetricThreshold bug_filing_thresholds = 10;

  // Next ID: 13
}


// ProjectMetadata provides data about the project that are mostly used in ui.
message ProjectMetadata {

  // Indicates the preferred display name for the project in the UI.
  // Deprecated: not used anymore.
  string display_name = 1;
}

// Settings related to metrics used to measure cluster impact.
message Metrics {
  message MetricOverride {
    // The id of the impact metric.
    //
    // Full list of available metrics here:
    // https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/analysis/internal/analysis/metrics/metrics.go
    string metric_id = 1;

    // Whether the metric should be selected by default.
    optional bool is_default = 2;

    // Controls the default sort order between metrics. By default,
    // a list will sort by the metric with the highest sort priority,
    // followed by the metric with second highest sort priority,
    // and so on.
    optional int32 sort_priority = 3;
  }

  // Overrides to the default metrics configuration for a project.
  repeated MetricOverride overrides = 1;
}

// Settings related to bug management.
message BugManagement {
  // Disables creation of comments on bugs when LUCI Analysis successfully
  // handles duplicate bugs by merging/updating failure association rules.
  //
  // This setting does not prevent the creation of comments in response
  // to errors handling duplicate bugs.
  bool disable_duplicate_bug_comments = 1;

  // The set of policies which control the (re-)opening, closure and
  // prioritization of bugs under the control of LUCI Analysis.
  repeated BugManagementPolicy policies = 2;

  // The default bug system to route new bugs to, when the bug system and
  // component could not be automatically detected from a test metadata.
  BugSystem default_bug_system = 3;

  // Buganizer-specific bug filing configuration.
  BuganizerProject buganizer = 4;

  // Monorail-specific bug filing configuration.
  MonorailProject monorail = 5;
}

// A bug management policy in LUCI Analysis.
//
// Bug management policies control when and how bugs are automatically
// opened, prioritised, and verified as fixed. Each policy has a user-visible
// identity in the UI and can post custom instructions on the bug.
//
// LUCI Analysis avoids filing multiple bugs for the same failures by
// allowing multiple policies to activate on the same failure association
// rule. The bug associated with a rule will only be verified if all policies
// have de-activated.
message BugManagementPolicy {
  // A unique identifier for the bug management policy.
  //
  // Policies are stateful in that LUCI Analysis tracks which bugs have met the
  // activation condition on the policy (and not since met the deactivation
  // condition).
  //
  // Changing this value changes the identity of the policy and hence results in
  // the activation state for the policy being lost for all bugs.
  //
  // Valid syntax: ^[a-z]([a-z0-9-]{0,62}[a-z0-9])?$. (Syntax designed to comply
  // with google.aip.dev/122 for resource IDs.)
  string id = 1;

  // The owners of the policy, who can be contacted if there are issues/concerns
  // about the policy. Each item in the list should be an @google.com email
  // address. At least one owner (preferably a group) is required.
  repeated string owners = 2;

  // A short one-line description for the problem the policy identifies, which
  // will appear on the UI and in bugs comments. This is a sentence fragment
  // and not a sentence, so please do NOT include a full stop and or starting
  // capital letter.
  //
  // For example, "test variant(s) are being exonerated in presubmit".
  string human_readable_name = 3;

  // The priority of the problem this policy defines.
  //
  // If:
  // - the priority of the bug associated with a rule
  //   differs from this priority, and
  // - the policy is activate on the rule (see `metrics`), and
  // - LUCI Analysis is controlling the priority of the bug
  //   (the "Update bug priority" switch on the rule is enabled),
  // the priority of the bug will be updated to match this priority.
  //
  // Where are there multiple policies active on the same rule,
  // the highest priority (of all active policies) will be used.
  //
  // For monorail projects, the buganizer priority will be converted to the
  // equivalent monorail priority (P0 is converted to Pri-0, P1 to Pri-1,
  // P2 to Pri-2, etc.) until monorail is turned down.
  BuganizerPriority priority = 4;

  // The set of metrics which will control activation of the bug-filing policy.
  // If a policy activates on a suggested cluster, a new bug will be filed.
  // If a policy activates on an existing rule cluster, the bug will be
  // updated.
  //
  // The policy will activate if the activation threshold is met on *ANY*
  // metric, and will de-activate only if the deactivation threshold is met
  // on *ALL* metrics.
  //
  // Activation on suggested clusters will be based on the metric values after
  // excluding failures for which a bug has already been filed. This is to
  // avoid duplicate bug filing.
  repeated Metric metrics = 5;

  // A metric used to control activation of a bug-filing policy.
  message Metric {
    // The identifier of the metric.
    //
    // Full list of available metrics here:
    // https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/analysis/internal/analysis/metrics/metrics.go
    string metric_id = 1;

    // The level at which the policy activates. Activation occurs if the
    // cluster impact meets or exceeds this threshold.
    // MUST imply deactivation_threshold.
    MetricThreshold activation_threshold = 2;

    // The minimum metric level at which the policy remains active.
    // Deactivation occcurs if the cluster impact is below the de-activation
    // threshold. Deactivation_threshold should be set significantly lower
    // than activation_threshold to prevent policies repeatedly activating
    // and deactivating due to noise in the data, e.g. less tests executed
    // on weekends.
    MetricThreshold deactivation_threshold = 3;
  }

  // Expanatory text of the problem the policy identified, shown on the
  // user interface when the user requests more information. Required.
  Explanation explanation = 6;

  // Content displayed on the user interface, to explain the problem and
  // guide a developer to fix it.
  message Explanation {
    // A longer human-readable description of the problem this policy
    // has identified, in HTML.
    //
    // For example, "Test variant(s) in this cluster are being exonerated
    // (ignored) in presubmit because they are too flaky or failing. This
    // means they are no longer effective at preventing the breakage of
    // the functionality the test(s) cover.".
    //
    // MUST be sanitised by UI before rendering. Sanitisation is only
    // required to support simple uses of the following tags: ul, li, a.
    string problem_html = 1;

    // A description of how a human should go about trying to fix the
    // problem, in HTML.
    //
    // For example, "<ul>
    // <li>View recent failures</li>
    // <li><a href="http://goto.google.com/demote-from-cq">Demote</a> the test from CQ</li>
    // </ul>"
    //
    // MUST be sanitised by UI before rendering. Sanitisation is only
    // required to support simple uses of the following tags: ul, li, a.
    string action_html = 2;
  }

  // Settings which affect the contents of a bug for a policy has activated.
  BugTemplate bug_template = 7;

  // Settings which affect the contents of a bug for a policy has activated.
  message BugTemplate {
    // Text to be included in the bug comment advising of the activation of the
    // the policy. Leave blank to post no comment.
    //
    // The text here will be interpreted as a template by the golang
    // template/text library (https://pkg.go.dev/text/template). The following
    // fields are available:
    // - RuleURL (string): The URL of the rule page.
    // - BugID (BugID): The system-specific bug identifier. This type
    //   exposes the following methods with no arguments:
    //   - IsBuganizer returns (bool) indicating if the bug is a buganizer bug.
    //   - IsMonorail returns (bool) indicating if the bug is a monorail bug.
    //   - MonorailProject returns (string, error) indicating the monorail
    //     project, if the bug is a monorail bug, and errors otherwise.
    //   - MonorailBugID returns (string, error) indicating the monorail
    //     bug ID, if the bug is a monorail bug, and errors otherwise.
    //   - BuganizerBugID returns (string, error) indicating the buganizer
    //     bug ID, if the bug is a buganizer bug, and errors otherwise.
    //
    // Model usage of BugID in a template:
    // ```
    // {{if .BugID.IsBuganizer}}Buganizer bug: {{.BugID.BuganizerBugID}}{{end}}
    // {{if .BugID.IsMonorail}}Monorail bug: {{.BugID.MonorailProject}}/{{.BugID.MonorailBugID}}{{end}}
    // ```
    //
    // As for functions, only the standard global functions are available, see:
    // https://pkg.go.dev/text/template#hdr-Functions
    string comment_template = 1;

    // Bug content options that are specific to Google issue tracker (Buganizer).
    Buganizer buganizer = 2;

    // Bug content options that are specific to monorail.
    Monorail monorail = 3;

    // TODO(meiring): Add assignee, cc list.

    // Policy configuration that is specific to Google issue tracker (Buganizer).
    message Buganizer {
      // The numeric ID of the buganizer hotlist to add the issue to. Optional.
      // The bug is added to the hostlist when the policy transitions to
      // activated. The issue is not removed from the hotlist if the policy is
      // deactivated, to avoid excessive bug updates.
      repeated int64 hotlists = 1;
    }

    // Policy configuration that is specific to monorail issue tracker.
    message Monorail {
      // The labels to apply to the bug.
      repeated string labels = 1;
    }
  }
}

// Deprecated. No longer has any effect. Retained for textproto
// compatibility only.
message ImpactThreshold {
  MetricThreshold test_results_failed = 4;
  MetricThreshold test_runs_failed = 5;
  MetricThreshold presubmit_runs_failed = 6;
  MetricThreshold critical_failures_exonerated = 7;
  optional int64 unexpected_failures_1d = 1;
  optional int64 unexpected_failures_3d = 2;
  optional int64 unexpected_failures_7d = 3;
}

// ImpactMetricThreshold specifies a condition on a cluster's impact metric.
message ImpactMetricThreshold {
  // The id of the impact metric.
  // e.g.
  // human-cls-failed-presubmit: The number of presubmit runs that failed.
  // critical-failures-exonerated: The number of test failures on critical
  //                               builders that were exonerated with an
  //                               exoneration reason other than NOT_CRITICAL.
  // test-runs-failed: The number of test runs that failed.
  //                   A test run (also known as a 'shard' (chromium) or
  //                   'task' (Chrome OS)) is considered failed if all tries of
  //                   test(s) in it unexpectedly failed. The failed test run is
  //                   attributed to the last failure of each of the test(s)
  //                   that failed on all tries.
  // failures: The number of test results that were unexpected failures.
  //
  // Full list of available metrics here:
  // https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/analysis/internal/analysis/metrics/metrics.go
  string metric_id = 1;

  // The thresholds against a metric.
  MetricThreshold threshold = 2;
}

// MetricThreshold specifies thresholds for a particular metric.
// The threshold is considered satisfied if any of the individual metric
// thresholds is met or exceeded (i.e. if multiple thresholds are set, they
// are combined using an OR-semantic). If no threshold is set, the threshold
// as a whole is unsatisfiable.
message MetricThreshold {
  // The threshold for one day.
  optional int64 one_day = 1;

  // The threshold for three day.
  optional int64 three_day = 2;

  // The threshold for seven days.
  optional int64 seven_day = 3;
}

// MonorailProject describes the configuration to use when filing bugs
// into a given monorail project.
message MonorailProject {
  // The monorail project being described.
  // E.g. "chromium".
  string project = 1;

  // The field values to use when creating new bugs.
  // For example, on chromium issue tracker, there is a manadatory
  // issue type field (field 10), which must be set to "Bug".
  repeated MonorailFieldValue default_field_values = 2;

  // The ID of the issue's priority field. You can find this by visiting
  // https://monorail-prod.appspot.com/p/<project>/adminLabels, scrolling
  // down to Custom fields and finding the ID of the field you wish to set.
  //
  // This field must support the values: "Pri-0", "Pri-1", "Pri-2", "Pri-3".
  int64 priority_field_id = 3;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.policies instead.
  repeated MonorailPriority priorities = 4;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only.
  int64 priority_hysteresis_percent = 5;

  // The prefix that should appear when displaying bugs from the
  // given bug tracking system. E.g. "crbug.com" or "fxbug.dev".
  // If no prefix is specified, only the bug number will appear.
  // Otherwise, the supplifed prefix will appear, followed by a
  // forward slash ("/"), followed by the bug number.
  // Valid prefixes match `^[a-z0-9\-.]{0,64}$`.
  string display_prefix = 6;

  // The preferred hostname to use in links to monorail. For example,
  // "bugs.chromium.org" or "bugs.fuchsia.dev".
  string monorail_hostname = 7;

  // Whether the Restrict-View-Google tag should be omitted on new
  // auto-filed bugs. This makes those bugs publically visible.
  // If unset, defaults to filing with Restrict-View-Google.
  bool file_without_restrict_view_google = 8;
}

// MonorailFieldValue describes a monorail field/value pair.
message MonorailFieldValue {
  // The ID of the field to set. You can find this by visiting
  // https://monorail-prod.appspot.com/p/<project>/adminLabels, scrolling
  // down to Custom fields and finding the ID of the field you wish to set.
  int64 field_id = 1;

  // The field value. Values are encoded according to the field type:
  // - Enumeration types: the string enumeration value (e.g. "Bug").
  // - Integer types: the integer, converted to a string (e.g. "1052").
  // - String types: the value, included verbatim.
  // - User types: the user's resource name (e.g. "users/2627516260").
  //   User IDs can be identified by looking at the people listing for a
  //   project:  https://monorail-prod.appspot.com/p/<project>/people/list.
  //   The User ID is included in the URL as u=<number> when clicking into
  //   the page for a particular user. For example, "user/3816576959" is
  //   https://monorail-prod.appspot.com/p/chromium/people/detail?u=3816576959.
  // - Date types: the number of seconds since epoch, as a string
  //   (e.g. "1609459200" for 1 January 2021).
  // - URL type: the URL value, as a string (e.g. "https://www.google.com/").
  //
  // The source of truth for mapping of field types to values is as
  // defined in the Monorail v3 API, found here:
  // https://source.chromium.org/chromium/infra/infra/+/main:appengine/monorail/api/v3/api_proto/issue_objects.proto?q=%22message%20FieldValue%22
  string value = 2;
}


// Deprecated. No longer has any effect. Retained for textproto
// compatibility only.
message MonorailPriority {
  string priority = 1;
  ImpactThreshold threshold = 2;
  repeated ImpactMetricThreshold thresholds = 3;
}

// Configurations per realm.
message RealmConfig {
  // Name of the realm.
  //
  // Must match `^[a-z0-9_\.\-/]{1,400}$`.
  // Must not contain the project part. I.e. for "chromium:ci" realm the value
  // here must be "ci".
  string name = 1;

  // Test variant analysis configurations for the realm.
  TestVariantAnalysisConfig test_variant_analysis = 2;
}

// Configuration for how test results are clustered.
message Clustering {
  // Rules used to cluster test results by test name.
  // The order of rules matters; the first matching rule will be used
  // to cluster a given test result.
  //
  // If no rule matches, the test results will be clustered on the
  // full test name. This corresponds approximately to the rule:
  // {
  //   name: "Full test name"
  //   pattern: "^(?P<testname>.*)$"
  //   like_template: "${testname}"
  // }
  repeated TestNameClusteringRule test_name_rules = 1;

  // Regular expressions used to mask out part of a failure reason
  // prior to clustering.
  //
  // The process of generating the clustering key is:
  // 1. All '%', '_' and '\' characters in the failure reason are
  //    escaped to generate a SQL LIKE expression that matches the
  //    failure reason literally.
  // 2. Regular expressions are run over the escaped failure reason
  //    one by one to identify parts of the failure reason to mask
  //    out (replace by a SQL LIKE wildcard match).
  // 3. The clustering key is used in the failure association rule
  //    of a newly field bug, or hashed to generate the clustering
  //    key.
  //
  // For regular expression run against the failure reason,
  // the part of the reason that matches the first (capturing)
  // subexpression is masked out in the reason cluster.
  // All non-overlapping matches are replaced.
  //
  // For example, given the masking expression:
  // "^\\[Fixture failure\\] (\\w+):"
  // The failure reason:
  // `[Fixture failure] myFixture: some_error`
  // will be escaped to (in step 1):
  // `[Fixture failure] myFixture: some\_error`
  // and will yield the following output after masking (step 2):
  // `[Fixture failure] %: some\_error`
  //
  // Masking expressions are applied in the order that they appear
  // in the list.
  repeated string reason_mask_patterns = 2;
}

// A rule used to cluster a test result by test name.
message TestNameClusteringRule {
  // A human-readable name for the rule. This should be unique for each rule.
  // This may be used by LUCI Analysis to explain why it chose to cluster the
  // test name in this way.
  string name = 1;

  // The regular expression describing which test names should be clustered
  // by this rule.
  //
  // Example.
  //   Assume our project uploads google test (gtest) results with the test
  //   name prefix "gtest://".
  //   If want to cluster value-parameterized google tests
  //   together based on the test suite and test case name (ignoring
  //   the value parameter), we may use a pattern like:
  //     "^gtest://(\w+/)?(?P<testcase>\w+\.\w+)/\w+$"
  //
  //   This will allow us to cluster test names like:
  //     "gtest://InstantiationOne/ColorSpaceTest.testNullTransform/0"
  //     "gtest://InstantiationOne/ColorSpaceTest.testNullTransform/1"
  //     "gtest://InstantiationTwo/ColorSpaceTest.testNullTransform/0"
  //   together.
  //
  //   See https://github.com/google/googletest/blob/main/docs/advanced.md#how-to-write-value-parameterized-tests
  //   to understand value-parameterised google tests.
  //
  // Use ?P<name> to name capture groups, so their values can be used in
  // like_template below.
  string pattern = 2;

  // The template used to generate a LIKE expression on test names
  // that defines the test name cluster identified by this rule.
  //
  // This like expression has two purposes:
  // (1) If the test name cluster is large enough to justify the
  //     creation of a bug cluster, the like expression is used to
  //     generate a failure association rule of the following form:
  //        test LIKE "<evaluated like_template>"
  // (2) A hash of the expression is used as the clustering key for the
  //     test name-based suggested cluster. This generally has the desired
  //     clustering behaviour, i.e. the parts of the test name which
  //     are important enough to included in the LIKE expression for (1)
  //     are also those on which clustering should occur.
  //
  // As is usual for LIKE expressions, the template can contain
  // the following operators to do wildcard matching:
  // * '%' for wildcard match of an arbitrary number of characters, and
  // * '_' for single character wildcard match.
  //
  // To match literal '%' or '_', escape the operator with a '\',
  // i.e. use "\%" or "\_" to match literal '%' and '_' respectively.
  // To match literal '\', you should use "\\".
  //
  // The template can refer to parts of the test name matched by
  // the rule pattern using ${name}, where name refers to the capture
  // group (see pattern). To insert the literal '$', the sequence '$$'
  // should be used.
  //
  // Example.
  //   Assume our project uploads google test (gtest) results with the test
  //   name prefix "gtest://". Further assume we used the pattern:
  //     "^gtest://(\w+/)?(?P<testcase>\w+\.\w+)/\w+$"
  //
  //   We might use the following like_template:
  //     "gtest://%${testcase}%"
  //
  //   When instantiated for a value-parameterised test, e.g.
  //   "gtest://InstantiationOne/ColorSpaceTest.testNullTransform/0",
  //   the result would be a failure association rule like:
  //     test LIKE "gtest://%ColorSpaceTest.testNullTransform%"
  //
  //   Note the use of ${testcase} to refer to the testname capture group
  //   specified in the pattern example.
  //
  //   See https://github.com/google/googletest/blob/main/docs/advanced.md#how-to-write-value-parameterized-tests
  //   to understand value-parameterised google tests.
  //
  // It is known that not all clusters can be precisely matched by
  // a LIKE expression. Nonetheless, LUCI Analysis prefers LIKE expressions
  // as they are easier to comprehend and modify by users, and in
  // most cases, the added precision is not required.
  //
  // As such, your rule should try to ensure the generated LIKE statement
  // captures your clustering logic as best it can. Your LIKE expression
  // MUST match all test names matched by your regex pattern, and MAY
  // capture additional test names (though this is preferably minimised,
  // to reduce differences between the suggested clusters and eventual
  // bug clusters).
  //
  // LUCI Analysis will automatically escape any '%' '_' and '\' in parts of
  // the matched test name before substitution to ensure captured parts
  // of the test name are matched literally and not interpreted.
  string like_template = 3;
}

// An enum that represents the bug filing system that the project uses.
enum BugSystem {
  // An unspecified bug system, Do not use, this will
  // break LUCI Analysis bug filing functionality.
  BUG_SYSTEM_UNSPECIFIED = 0;
  // Use Monorail to file bugs.
  MONORAIL = 1;
  // Use Buganizer to file bugs.
  BUGANIZER = 2;
}

// This enum represents the Buganizer priorities.
// It is equivalent to the one in Buganizer API.
enum BuganizerPriority {
  // Priority unspecified, Do not use this value.
  BUGANIZER_PRIORITY_UNSPECIFIED = 0;
  // P0, Highest priority.
  P0 = 1;
  P1 = 2;
  P2 = 3;
  P3 = 4;
  P4 = 5;
}

// Defines the required details for a Buganizer component.
message BuganizerComponent {
  // The id of the component that we will use to file bugs in.
  int64 id = 1;
}

// The Buganizer configuration, this should only be
// used when the bug tracking system ins Buganizer.
message BuganizerProject {
  // The default Buganizer component.
  // This component will be used if we failed to find
  // a component for a cluster.
  BuganizerComponent default_component = 1;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only.
  int64 priority_hysteresis_percent = 2;

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only.
  message PriorityMapping {
    BuganizerPriority priority = 1;
    ImpactThreshold threshold = 2;
    repeated ImpactMetricThreshold thresholds = 3;
  }

  // Deprecated. No longer has any effect. Retained for textproto
  // compatibility only. Use bug_management.policies instead.
  repeated PriorityMapping priority_mappings = 3;

  // Whether the LIMIT_VIEW_TRUSTED access level should be omitted
  // on new auto-filed bugs and LIMIT_NONE access level should be set.
  // This makes those bugs visible to all those who can see bugs
  // in a given component.
  bool file_without_limit_view_trusted = 4;
}

// Criteria used to determine test stability. This criteria is used
// to inform test exoneration in presubmit via the
// TestVariants.QueryStability RPC.
//
// Criteria is applied using a data source which contains
// the last 14 days' of test result data for all test variants,
// with certain filterings applied.
//
// See go/luci-exoneration-v2 as well each criteria below for more details.
message TestStabilityCriteria {
  // The failure rate criteria to apply. Mandatory.
  FailureRateCriteria failure_rate = 1;

  // The failure rate criteria detects consistently failing
  // and highly flaky tests (e.g. 95%+ failing) by looking for
  // a high number of failures at the queried position of the
  // test's history.
  //
  // The criteria obtains from the last 14 days' of filtered test data
  // a set of (up to) 20 test runs centered on the queried commit
  // position (10 prior and 10 after) and applies criteria
  // to this in various ways.
  // The 20 test runs are sorted by commit position and then time.
  //
  // See go/luci-exoneration-v2 for more detail.
  message FailureRateCriteria {
      // The number of unexpected test runs that must be
      // found in a sliding window of size 10 containing the
      // queried position to begin exoneration.
      // 6 is a good starting value.
      //
      // The criteria is applied over sliding windows of size
      // 10 around the query position. Assuming the full 20 test
      // runs are obtained, this means 11 window positions are considered.
      // If any window satisifes the threshold, the criteria is met
      // and the test is considered unstable.
      //
      // In the event that 10 test runs cannot be found in the last
      // 14 days of test history, a window sized to the available
      // test runs is used but the criteria is not scaled.
      int32 failure_threshold = 1;

      // The number of consecutive unexpected test runs, which if
      // present at the leading or trailing part of the (up to) 20
      // test verdicts, will trigger exoneration.
      // 3 is a good starting value.
      //
      // The consecutive failures must also touch the query position.
      //
      // This is designed to create a fast path to exoneration for
      // 100% failing tests which produce a strong and consistent
      // failing signal, leveraging the statistical significance
      // of consecutive failures. If this threshold is met,
      // the failure_threshold above does NOT need to be met.
      //
      // E.g. the following scenario WILL trigger this criteria for
      // a threshold of four or less.
      //
      // History: >F F F F< P P P P P P P
      //            ^
      //            Query position
      //
      // The following scenario WILL NOT trigger this criteria:
      //
      // History:>P F F F F< P P P P P P P
      //              ^
      //              Query position
      //
      // (N.B. Direction of history is irrelevant as criteria is
      // applied symmetrically. Either the left or right could
      // represent 'later' by commit position.)
      int32 consecutive_failure_threshold = 2;
  }

  // The flake rate criteria to apply. Mandatory.
  FlakeRateCriteria flake_rate = 2;

  // The flake rate criteria detects flaky tests by looking for
  // examples where a test has obtained expected and unexpected
  // test runs for the same sources under test.
  //
  // If there are more flaky source verdicts found than a threshold,
  // the test is considered flaky.
  //
  // The analysis window is all source verdicts for 7 days' worth
  // of commit positions either side of the queried position.
  // The conversion between time and commit position is discussed
  // in go/luci-exoneration-v2.
  //
  // In the event that an unsatisfactory number of source positions
  // are found using this method, the window is enlarged to possibly
  // include any verdict in the last 14 days. This is to improve
  // detection performance on tests with a low volume of results.
  message FlakeRateCriteria {
    // The minimum number of source verdicts desired
    // for the analysis window.
    //
    // As standard, all source verdicts for sources
    // +/- 7 days from the queried position are used.
    //
    // However, if the number of verdicts is not equal
    // to or greater than min_window, all source verdicts
    // from the last 14 days will be used. This is designed
    // to prioritise adequate flake detection performance
    // for test variants with low result volumes, at the
    // cost of data recency.
    //
    // If the number of source verdicts in the last 14 days
    // is less than min_window, then whatever source verdicts
    // are available are still used.
    //
    // 100 is a good starting value.
    int32 min_window = 1;

    // The minimum number of flaky source verdicts required
    // to trigger the criteria. 2 is a good starting value.
    int32 flake_threshold = 2;

    // The minimum flake rate required to trigger the criteria,
    // as a proportion of all source verdicts. This must be a
    // value between 0.0 and 1.0.
    // 0.01 (1%) is a good starting value.
    //
    // Both flake_threshold AND the flake_rate_threshold must be met
    // for a test to be considered unstable.
    //
    // Note that not even the most flaky (50% flaky) test would
    // be expected to produce more than a 25% flake rate if
    // failures are retried once. This is because its expected
    // outcomes are:
    // - Pass on first try = 50%
    // - Fail on first try, pass on second try = 25% (flaky)
    // - Fail on both tries = 25%
    double flake_rate_threshold = 3;
  }
}