go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/internal/clustering/ingestion/ingest_test.go

// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ingestion

import (
	"encoding/hex"
	"fmt"
	"sort"
	"testing"
	"time"

	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/durationpb"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/gae/impl/memory"
	rdbpb "go.chromium.org/luci/resultdb/proto/v1"
	"go.chromium.org/luci/server/caching"

	"go.chromium.org/luci/analysis/internal/analysis"
	"go.chromium.org/luci/analysis/internal/analysis/clusteredfailures"
	"go.chromium.org/luci/analysis/internal/clustering"
	"go.chromium.org/luci/analysis/internal/clustering/algorithms"
	"go.chromium.org/luci/analysis/internal/clustering/algorithms/failurereason"
	"go.chromium.org/luci/analysis/internal/clustering/algorithms/rulesalgorithm"
	"go.chromium.org/luci/analysis/internal/clustering/algorithms/testname"
	"go.chromium.org/luci/analysis/internal/clustering/chunkstore"
	clusteringpb "go.chromium.org/luci/analysis/internal/clustering/proto"
	"go.chromium.org/luci/analysis/internal/clustering/rules"
	"go.chromium.org/luci/analysis/internal/config"
	"go.chromium.org/luci/analysis/internal/config/compiledcfg"
	"go.chromium.org/luci/analysis/internal/testutil"
	bqpb "go.chromium.org/luci/analysis/proto/bq"
	configpb "go.chromium.org/luci/analysis/proto/config"
	pb "go.chromium.org/luci/analysis/proto/v1"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/common/testing/assertions"
)

func TestIngest(t *testing.T) {
	Convey(`With Ingestor`, t, func() {
		ctx := testutil.IntegrationTestContext(t)
		ctx = caching.WithEmptyProcessCache(ctx) // For rules cache.
		ctx = memory.Use(ctx)                    // For project config in datastore.

		chunkStore := chunkstore.NewFakeClient()
		clusteredFailures := clusteredfailures.NewFakeClient()
		analysis := analysis.NewClusteringHandler(clusteredFailures)
		ingestor := New(chunkStore, analysis)

		opts := Options{
			TaskIndex:     1,
			Project:       "chromium",
			PartitionTime: time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC),
			Realm:         "chromium:ci",
			InvocationID:  "build-123456790123456",
			PresubmitRun: &PresubmitRun{
				ID:     &pb.PresubmitRunId{System: "luci-cv", Id: "cq-run-123"},
				Owner:  "automation",
				Mode:   pb.PresubmitRunMode_FULL_RUN,
				Status: pb.PresubmitRunStatus_PRESUBMIT_RUN_STATUS_FAILED,
			},
			BuildStatus:            pb.BuildStatus_BUILD_STATUS_FAILURE,
			BuildCritical:          true,
			BuildGardenerRotations: []string{"gardener-rotation1", "gardener-rotation2"},
		}
		testIngestion := func(input []TestVerdict, expectedCFs []*bqpb.ClusteredFailureRow) {
			err := ingestor.Ingest(ctx, opts, input)
			So(err, ShouldBeNil)

			insertions := clusteredFailures.Insertions
			So(len(insertions), ShouldEqual, len(expectedCFs))

			// Sort both actuals and expectations by key so that we compare corresponding rows.
			sortClusteredFailures(insertions)
			sortClusteredFailures(expectedCFs)
			for i, exp := range expectedCFs {
				actual := insertions[i]
				So(actual, ShouldNotBeNil)

				// Chunk ID and index is assigned by ingestion.
				copyExp := proto.Clone(exp).(*bqpb.ClusteredFailureRow)
				So(actual.ChunkId, ShouldNotBeEmpty)
				So(actual.ChunkIndex, ShouldBeGreaterThanOrEqualTo, 1)
				copyExp.ChunkId = actual.ChunkId
				copyExp.ChunkIndex = actual.ChunkIndex

				// LastUpdated time is assigned by Spanner.
				So(actual.LastUpdated, ShouldNotBeZeroValue)
				copyExp.LastUpdated = actual.LastUpdated

				So(actual, ShouldResembleProto, copyExp)
			}
		}

		// This rule should match failures used in this test.
		rule := rules.NewRule(100).WithProject(opts.Project).WithRuleDefinition(`reason LIKE "Failure reason%"`).Build()
		err := rules.SetForTesting(ctx, []*rules.Entry{
			rule,
		})
		So(err, ShouldBeNil)

		// Setup clustering configuration
		projectCfg := &configpb.ProjectConfig{
			Clustering:  algorithms.TestClusteringConfig(),
			LastUpdated: timestamppb.New(time.Date(2020, time.January, 5, 0, 0, 0, 1, time.UTC)),
		}
		projectCfgs := map[string]*configpb.ProjectConfig{
			"chromium": projectCfg,
		}
		So(config.SetTestProjectConfig(ctx, projectCfgs), ShouldBeNil)

		cfg, err := compiledcfg.NewConfig(projectCfg)
		So(err, ShouldBeNil)

		Convey(`Ingest one failure`, func() {
			const uniqifier = 1
			const testRunCount = 1
			const resultsPerTestRun = 1
			tv := newTestVerdict(uniqifier, testRunCount, resultsPerTestRun, nil)
			tvs := []TestVerdict{tv}

			// Expect the test result to be clustered by both reason and test name.
			const testRunNum = 0
			const resultNum = 0
			regexpCF := expectedClusteredFailure(uniqifier, testRunCount, testRunNum, resultsPerTestRun, resultNum)
			setRegexpClustered(cfg, regexpCF)
			testnameCF := expectedClusteredFailure(uniqifier, testRunCount, testRunNum, resultsPerTestRun, resultNum)
			setTestNameClustered(cfg, testnameCF)
			ruleCF := expectedClusteredFailure(uniqifier, testRunCount, testRunNum, resultsPerTestRun, resultNum)
			setRuleClustered(ruleCF, rule)
			expectedCFs := []*bqpb.ClusteredFailureRow{regexpCF, testnameCF, ruleCF}

			Convey(`Unexpected failure`, func() {
				tv.Verdict.Results[0].Result.Status = rdbpb.TestStatus_FAIL
				tv.Verdict.Results[0].Result.Expected = false

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
			Convey(`Expected failure`, func() {
				tv.Verdict.Results[0].Result.Status = rdbpb.TestStatus_FAIL
				tv.Verdict.Results[0].Result.Expected = true

				// Expect no test results ingested for an expected
				// failure.
				expectedCFs = nil

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 0)
			})
			Convey(`Unexpected pass`, func() {
				tv.Verdict.Results[0].Result.Status = rdbpb.TestStatus_PASS
				tv.Verdict.Results[0].Result.Expected = false

				// Expect no test results ingested for a passed test
				// (even if unexpected).
				expectedCFs = nil
				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 0)
			})
			Convey(`Unexpected skip`, func() {
				tv.Verdict.Results[0].Result.Status = rdbpb.TestStatus_SKIP
				tv.Verdict.Results[0].Result.Expected = false

				// Expect no test results ingested for a skipped test
				// (even if unexpected).
				expectedCFs = nil

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 0)
			})
			Convey(`Failure with no tags`, func() {
				// Tests are allowed to have no tags.
				tv.Verdict.Results[0].Result.Tags = nil

				for _, cf := range expectedCFs {
					cf.Tags = nil
					cf.BugTrackingComponent = nil
				}

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
			Convey(`Failure without variant`, func() {
				// Tests are allowed to have no variant.
				tv.Verdict.Variant = nil
				tv.Verdict.Results[0].Result.Variant = nil

				for _, cf := range expectedCFs {
					cf.Variant = nil
				}

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
			Convey(`Failure without failure reason`, func() {
				// Failures may not have a failure reason.
				tv.Verdict.Results[0].Result.FailureReason = nil
				testnameCF.FailureReason = nil

				// As the test result does not match any rules, the
				// test result is included in the suggested cluster
				// with high priority.
				testnameCF.IsIncludedWithHighPriority = true
				expectedCFs = []*bqpb.ClusteredFailureRow{testnameCF}

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
			Convey(`Failure without presubmit run`, func() {
				opts.PresubmitRun = nil
				for _, cf := range expectedCFs {
					cf.PresubmitRunId = nil
					cf.PresubmitRunMode = ""
					cf.PresubmitRunOwner = ""
					cf.PresubmitRunStatus = ""
					cf.BuildCritical = false
				}

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
			Convey(`Failure with multiple exoneration`, func() {
				tv.Verdict.Exonerations = []*rdbpb.TestExoneration{
					{
						Name:            fmt.Sprintf("invocations/testrun-mytestrun/tests/test-name-%v/exonerations/exon-1", uniqifier),
						TestId:          tv.Verdict.TestId,
						Variant:         proto.Clone(tv.Verdict.Variant).(*rdbpb.Variant),
						VariantHash:     "hash",
						ExonerationId:   "exon-1",
						ExplanationHtml: "<p>Some description</p>",
						Reason:          rdbpb.ExonerationReason_OCCURS_ON_MAINLINE,
					},
					{
						Name:            fmt.Sprintf("invocations/testrun-mytestrun/tests/test-name-%v/exonerations/exon-1", uniqifier),
						TestId:          tv.Verdict.TestId,
						Variant:         proto.Clone(tv.Verdict.Variant).(*rdbpb.Variant),
						VariantHash:     "hash",
						ExonerationId:   "exon-1",
						ExplanationHtml: "<p>Some description</p>",
						Reason:          rdbpb.ExonerationReason_OCCURS_ON_OTHER_CLS,
					},
				}

				for _, cf := range expectedCFs {
					cf.Exonerations = []*bqpb.ClusteredFailureRow_TestExoneration{
						{
							Reason: pb.ExonerationReason_OCCURS_ON_MAINLINE,
						}, {
							Reason: pb.ExonerationReason_OCCURS_ON_OTHER_CLS,
						},
					}
				}

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
			Convey(`Failure with only suggested clusters`, func() {
				reason := &pb.FailureReason{
					PrimaryErrorMessage: "Should not match rule",
				}
				tv.Verdict.Results[0].Result.FailureReason = &rdbpb.FailureReason{
					PrimaryErrorMessage: "Should not match rule",
				}
				testnameCF.FailureReason = reason
				regexpCF.FailureReason = reason

				// Recompute the cluster ID to reflect the different
				// failure reason.
				setRegexpClustered(cfg, regexpCF)

				// As the test result does not match any rules, the
				// test result should be included in the suggested clusters
				// with high priority.
				testnameCF.IsIncludedWithHighPriority = true
				regexpCF.IsIncludedWithHighPriority = true
				expectedCFs = []*bqpb.ClusteredFailureRow{testnameCF, regexpCF}

				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})

			Convey(`Failure with bug component metadata`, func() {
				Convey(`With monorail bug system`, func() {
					tv.Verdict.TestMetadata.BugComponent = &rdbpb.BugComponent{
						System: &rdbpb.BugComponent_Monorail{
							Monorail: &rdbpb.MonorailComponent{
								Project: "chromium",
								Value:   "Blink>Component",
							},
						},
					}
					for _, cf := range expectedCFs {
						cf.BugTrackingComponent.System = "monorail"
						cf.BugTrackingComponent.Component = "Blink>Component"
					}
					testIngestion(tvs, expectedCFs)
					So(len(chunkStore.Contents), ShouldEqual, 1)
				})
				Convey(`With Buganizer bug system`, func() {
					tv.Verdict.TestMetadata.BugComponent = &rdbpb.BugComponent{
						System: &rdbpb.BugComponent_IssueTracker{
							IssueTracker: &rdbpb.IssueTrackerComponent{
								ComponentId: 12345,
							},
						},
					}
					for _, cf := range expectedCFs {
						cf.BugTrackingComponent.System = "buganizer"
						cf.BugTrackingComponent.Component = "12345"
					}
					testIngestion(tvs, expectedCFs)
					So(len(chunkStore.Contents), ShouldEqual, 1)
				})
				Convey(`No BugComponent metadata, but public_buganizer_component tag present`, func() {
					tv.Verdict.TestMetadata.BugComponent = nil

					for _, result := range tv.Verdict.Results {
						result.Result.Tags = []*rdbpb.StringPair{
							{
								Key:   "public_buganizer_component",
								Value: "654321",
							},
						}
					}
					for _, cf := range expectedCFs {
						cf.BugTrackingComponent.System = "buganizer"
						cf.BugTrackingComponent.Component = "654321"
						cf.Tags = []*pb.StringPair{
							{
								Key:   "public_buganizer_component",
								Value: "654321",
							},
						}
					}
					testIngestion(tvs, expectedCFs)
					So(len(chunkStore.Contents), ShouldEqual, 1)
				})
				Convey(`No BugComponent metadata, both public_buganizer_component and monorail_component present`, func() {
					tv.Verdict.TestMetadata.BugComponent = nil

					for _, result := range tv.Verdict.Results {
						result.Result.Tags = []*rdbpb.StringPair{
							{
								Key:   "monorail_component",
								Value: "Component>MyComponent",
							},
							{
								Key:   "public_buganizer_component",
								Value: "654321",
							},
						}
					}
					for _, cf := range expectedCFs {
						cf.Tags = []*pb.StringPair{
							{
								Key:   "monorail_component",
								Value: "Component>MyComponent",
							},
							{
								Key:   "public_buganizer_component",
								Value: "654321",
							},
						}
					}

					Convey("With monorail as preferred system", func() {
						opts.PreferBuganizerComponents = false

						for _, cf := range expectedCFs {
							cf.BugTrackingComponent.System = "monorail"
							cf.BugTrackingComponent.Component = "Component>MyComponent"
						}
						testIngestion(tvs, expectedCFs)
						So(len(chunkStore.Contents), ShouldEqual, 1)
					})
					Convey("With buganizer as preferred system", func() {
						opts.PreferBuganizerComponents = true

						for _, cf := range expectedCFs {
							cf.BugTrackingComponent.System = "buganizer"
							cf.BugTrackingComponent.Component = "654321"
						}
						testIngestion(tvs, expectedCFs)
						So(len(chunkStore.Contents), ShouldEqual, 1)
					})
				})
			})
			Convey(`Failure with no sources`, func() {
				tv.Sources = nil
				for _, cf := range expectedCFs {
					cf.Sources = nil
					cf.SourceRef = nil
					cf.SourceRefHash = "<fix me>"
				}
				// No sources also means no test variant branch analysis.
				tv.TestVariantBranch = nil
			})
		})
		Convey(`Ingest multiple failures`, func() {
			const uniqifier = 1
			const testRunsPerVariant = 2
			const resultsPerTestRun = 2
			tv := newTestVerdict(uniqifier, testRunsPerVariant, resultsPerTestRun, nil)
			tvs := []TestVerdict{tv}

			// Setup a scenario as follows:
			// - A test was run four times in total, consisting of two test
			//   runs with two tries each.
			// - The test failed on all tries.
			var expectedCFs []*bqpb.ClusteredFailureRow
			var expectedCFsByTestRun [][]*bqpb.ClusteredFailureRow
			for t := 0; t < testRunsPerVariant; t++ {
				var testRunExp []*bqpb.ClusteredFailureRow
				for j := 0; j < resultsPerTestRun; j++ {
					regexpCF := expectedClusteredFailure(uniqifier, testRunsPerVariant, t, resultsPerTestRun, j)
					setRegexpClustered(cfg, regexpCF)
					testnameCF := expectedClusteredFailure(uniqifier, testRunsPerVariant, t, resultsPerTestRun, j)
					setTestNameClustered(cfg, testnameCF)
					ruleCF := expectedClusteredFailure(uniqifier, testRunsPerVariant, t, resultsPerTestRun, j)
					setRuleClustered(ruleCF, rule)
					testRunExp = append(testRunExp, regexpCF, testnameCF, ruleCF)
				}
				expectedCFsByTestRun = append(expectedCFsByTestRun, testRunExp)
				expectedCFs = append(expectedCFs, testRunExp...)
			}

			// Expectation: all test results show both the test run and
			// invocation blocked by failures.
			for _, exp := range expectedCFs {
				exp.IsIngestedInvocationBlocked = true
				exp.IsTestRunBlocked = true
			}

			Convey(`Some test runs blocked and presubmit run not blocked`, func() {
				// Let the last retry of the last test run pass.
				tv.Verdict.Results[testRunsPerVariant*resultsPerTestRun-1].Result.Status = rdbpb.TestStatus_PASS
				// Drop the expected clustered failures for the last test result.
				expectedCFs = expectedCFs[0 : (testRunsPerVariant*resultsPerTestRun-1)*3]

				// First test run should be blocked.
				for _, exp := range expectedCFsByTestRun[0] {
					exp.IsIngestedInvocationBlocked = false
					exp.IsTestRunBlocked = true
				}
				// Last test run should not be blocked.
				for _, exp := range expectedCFsByTestRun[testRunsPerVariant-1] {
					exp.IsIngestedInvocationBlocked = false
					exp.IsTestRunBlocked = false
				}
				testIngestion(tvs, expectedCFs)
				So(len(chunkStore.Contents), ShouldEqual, 1)
			})
		})
		Convey(`Ingest many failures`, func() {
			var tvs []TestVerdict
			var expectedCFs []*bqpb.ClusteredFailureRow

			const variantCount = 20
			const testRunsPerVariant = 10
			const resultsPerTestRun = 10
			for uniqifier := 0; uniqifier < variantCount; uniqifier++ {
				tv := newTestVerdict(uniqifier, testRunsPerVariant, resultsPerTestRun, nil)
				tvs = append(tvs, tv)
				for t := 0; t < testRunsPerVariant; t++ {
					for j := 0; j < resultsPerTestRun; j++ {
						regexpCF := expectedClusteredFailure(uniqifier, testRunsPerVariant, t, resultsPerTestRun, j)
						setRegexpClustered(cfg, regexpCF)
						testnameCF := expectedClusteredFailure(uniqifier, testRunsPerVariant, t, resultsPerTestRun, j)
						setTestNameClustered(cfg, testnameCF)
						ruleCF := expectedClusteredFailure(uniqifier, testRunsPerVariant, t, resultsPerTestRun, j)
						setRuleClustered(ruleCF, rule)
						expectedCFs = append(expectedCFs, regexpCF, testnameCF, ruleCF)
					}
				}
			}
			// Verify more than one chunk is ingested.
			testIngestion(tvs, expectedCFs)
			So(len(chunkStore.Contents), ShouldBeGreaterThan, 1)
		})
	})
}

func setTestNameClustered(cfg *compiledcfg.ProjectConfig, e *bqpb.ClusteredFailureRow) {
	e.ClusterAlgorithm = testname.AlgorithmName
	e.ClusterId = hex.EncodeToString((&testname.Algorithm{}).Cluster(cfg, &clustering.Failure{
		TestID: e.TestId,
	}))
}

func setRegexpClustered(cfg *compiledcfg.ProjectConfig, e *bqpb.ClusteredFailureRow) {
	e.ClusterAlgorithm = failurereason.AlgorithmName
	e.ClusterId = hex.EncodeToString((&failurereason.Algorithm{}).Cluster(cfg, &clustering.Failure{
		Reason: &pb.FailureReason{PrimaryErrorMessage: e.FailureReason.PrimaryErrorMessage},
	}))
}

func setRuleClustered(e *bqpb.ClusteredFailureRow, rule *rules.Entry) {
	e.ClusterAlgorithm = rulesalgorithm.AlgorithmName
	e.ClusterId = rule.RuleID
	e.IsIncludedWithHighPriority = true
}

func sortClusteredFailures(cfs []*bqpb.ClusteredFailureRow) {
	sort.Slice(cfs, func(i, j int) bool {
		return clusteredFailureKey(cfs[i]) < clusteredFailureKey(cfs[j])
	})
}

func clusteredFailureKey(cf *bqpb.ClusteredFailureRow) string {
	return fmt.Sprintf("%q/%q/%q/%q", cf.ClusterAlgorithm, cf.ClusterId, cf.TestResultSystem, cf.TestResultId)
}

func newTestVerdict(uniqifier, testRunCount, resultsPerTestRun int, bugComponent *rdbpb.BugComponent) TestVerdict {
	testID := fmt.Sprintf("ninja://test_name/%v", uniqifier)
	variant := &rdbpb.Variant{
		Def: map[string]string{
			"k1": "v1",
		},
	}
	rdbVerdict := &rdbpb.TestVariant{
		TestId:       testID,
		Variant:      variant,
		VariantHash:  "hash",
		Status:       rdbpb.TestVariantStatus_UNEXPECTED,
		Exonerations: nil,
		TestMetadata: &rdbpb.TestMetadata{
			BugComponent: bugComponent,
		},
	}
	for i := 0; i < testRunCount; i++ {
		for j := 0; j < resultsPerTestRun; j++ {
			tr := newTestResult(uniqifier, i, j)
			// Test ID, Variant, VariantHash are not populated on the test
			// results of a Test Variant as it is present on the parent record.
			tr.TestId = ""
			tr.Variant = nil
			tr.VariantHash = ""
			rdbVerdict.Results = append(rdbVerdict.Results, &rdbpb.TestResultBundle{Result: tr})
		}
	}
	return TestVerdict{
		Verdict: rdbVerdict,
		Sources: testSources(),
		TestVariantBranch: &clusteringpb.TestVariantBranch{
			FlakyVerdicts_24H:      111,
			UnexpectedVerdicts_24H: 222,
			TotalVerdicts_24H:      555,
		},
	}
}

func newTestResult(uniqifier, testRunNum, resultNum int) *rdbpb.TestResult {
	return newFakeTestResult(uniqifier, testRunNum, resultNum)
}

func newFakeTestResult(uniqifier, testRunNum, resultNum int) *rdbpb.TestResult {
	resultID := fmt.Sprintf("result-%v-%v", testRunNum, resultNum)
	return &rdbpb.TestResult{
		Name:        fmt.Sprintf("invocations/testrun-%v/tests/test-name-%v/results/%s", testRunNum, uniqifier, resultID),
		ResultId:    resultID,
		Expected:    false,
		Status:      rdbpb.TestStatus_CRASH,
		SummaryHtml: "<p>Some SummaryHTML</p>",
		StartTime:   timestamppb.New(time.Date(2022, time.February, 12, 0, 0, 0, 0, time.UTC)),
		Duration:    durationpb.New(time.Second * 10),
		Tags: []*rdbpb.StringPair{
			{
				Key:   "monorail_component",
				Value: "Component>MyComponent",
			},
		},
		FailureReason: &rdbpb.FailureReason{
			PrimaryErrorMessage: "Failure reason.",
		},
	}
}

func testSources() *pb.Sources {
	result := &pb.Sources{
		GitilesCommit: &pb.GitilesCommit{
			Host:       "chromium.googlesource.com",
			Project:    "infra/infra",
			Ref:        "refs/heads/main",
			CommitHash: "1234567890abcdefabcd1234567890abcdefabcd",
			Position:   12345,
		},
		IsDirty: true,
		Changelists: []*pb.GerritChange{
			{
				Host:     "chromium-review.googlesource.com",
				Project:  "myproject",
				Change:   87654,
				Patchset: 321,
			},
		},
	}
	return result
}

func expectedClusteredFailure(uniqifier, testRunCount, testRunNum, resultsPerTestRun, resultNum int) *bqpb.ClusteredFailureRow {
	resultID := fmt.Sprintf("result-%v-%v", testRunNum, resultNum)
	return &bqpb.ClusteredFailureRow{
		ClusterAlgorithm:           "", // Determined by clustering algorithm.
		ClusterId:                  "", // Determined by clustering algorithm.
		TestResultSystem:           "resultdb",
		TestResultId:               fmt.Sprintf("invocations/testrun-%v/tests/test-name-%v/results/%s", testRunNum, uniqifier, resultID),
		LastUpdated:                nil, // Only known at runtime, Spanner commit timestamp.
		Project:                    "chromium",
		PartitionTime:              timestamppb.New(time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)),
		IsIncluded:                 true,
		IsIncludedWithHighPriority: false,

		ChunkId:    "",
		ChunkIndex: 0, // To be set by caller as needed.

		Realm:  "chromium:ci",
		TestId: fmt.Sprintf("ninja://test_name/%v", uniqifier),
		Tags: []*pb.StringPair{
			{
				Key:   "monorail_component",
				Value: "Component>MyComponent",
			},
		},
		Variant: []*pb.StringPair{
			{
				Key:   "k1",
				Value: "v1",
			},
		},
		VariantHash:   "hash",
		FailureReason: &pb.FailureReason{PrimaryErrorMessage: "Failure reason."},
		BugTrackingComponent: &pb.BugTrackingComponent{
			System:    "monorail",
			Component: "Component>MyComponent",
		},
		StartTime:    timestamppb.New(time.Date(2022, time.February, 12, 0, 0, 0, 0, time.UTC)),
		Duration:     10.0,
		Exonerations: nil,

		PresubmitRunId:                &pb.PresubmitRunId{System: "luci-cv", Id: "cq-run-123"},
		PresubmitRunOwner:             "automation",
		PresubmitRunMode:              "FULL_RUN", // pb.PresubmitRunMode_FULL_RUN
		PresubmitRunStatus:            "FAILED",   // pb.PresubmitRunStatus_PRESUBMIT_RUN_STATUS_FAILED,
		BuildStatus:                   "FAILURE",  // pb.BuildStatus_BUILD_STATUS_FAILURE
		BuildCritical:                 true,
		IngestedInvocationId:          "build-123456790123456",
		IngestedInvocationResultIndex: int64(testRunNum*resultsPerTestRun + resultNum),
		IngestedInvocationResultCount: int64(testRunCount * resultsPerTestRun),
		IsIngestedInvocationBlocked:   true,
		TestRunId:                     fmt.Sprintf("testrun-%v", testRunNum),
		TestRunResultIndex:            int64(resultNum),
		TestRunResultCount:            int64(resultsPerTestRun),
		IsTestRunBlocked:              true,

		SourceRefHash: `923c0d1af67f8ef8`,
		SourceRef: &pb.SourceRef{
			System: &pb.SourceRef_Gitiles{
				Gitiles: &pb.GitilesRef{
					Host:    "chromium.googlesource.com",
					Project: "infra/infra",
					Ref:     "refs/heads/main",
				},
			},
		},
		Sources:                testSources(),
		BuildGardenerRotations: []string{"gardener-rotation1", "gardener-rotation2"},
		TestVariantBranch: &bqpb.ClusteredFailureRow_TestVariantBranch{
			FlakyVerdicts_24H:      111,
			UnexpectedVerdicts_24H: 222,
			TotalVerdicts_24H:      555,
		},
	}
}