go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/bisection/culpritaction/revertculprit/revertculprit_test.go (about)

     1  // Copyright 2022 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package revertculprit
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"testing"
    21  	"time"
    22  
    23  	"github.com/golang/mock/gomock"
    24  	. "github.com/smartystreets/goconvey/convey"
    25  	"google.golang.org/grpc/codes"
    26  	"google.golang.org/grpc/status"
    27  	"google.golang.org/protobuf/types/known/timestamppb"
    28  
    29  	"go.chromium.org/luci/bisection/internal/config"
    30  	"go.chromium.org/luci/bisection/internal/gerrit"
    31  	"go.chromium.org/luci/bisection/internal/rotationproxy"
    32  	"go.chromium.org/luci/bisection/model"
    33  	configpb "go.chromium.org/luci/bisection/proto/config"
    34  	pb "go.chromium.org/luci/bisection/proto/v1"
    35  	"go.chromium.org/luci/bisection/util"
    36  	"go.chromium.org/luci/bisection/util/datastoreutil"
    37  	"go.chromium.org/luci/bisection/util/testutil"
    38  
    39  	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
    40  	"go.chromium.org/luci/common/clock"
    41  	"go.chromium.org/luci/common/clock/testclock"
    42  	"go.chromium.org/luci/common/proto"
    43  	gerritpb "go.chromium.org/luci/common/proto/gerrit"
    44  	. "go.chromium.org/luci/common/testing/assertions"
    45  	"go.chromium.org/luci/common/tsmon"
    46  	"go.chromium.org/luci/gae/impl/memory"
    47  	"go.chromium.org/luci/gae/service/datastore"
    48  )
    49  
    50  func TestRevertCulprit(t *testing.T) {
    51  	t.Parallel()
    52  
    53  	Convey("RevertCulprit", t, func() {
    54  		ctx := memory.Use(context.Background())
    55  		testutil.UpdateIndices(ctx)
    56  
    57  		// Set test clock
    58  		cl := testclock.New(testclock.TestTimeUTC)
    59  		ctx = clock.Set(ctx, cl)
    60  
    61  		// Setup tsmon
    62  		ctx, _ = tsmon.WithDummyInMemory(ctx)
    63  
    64  		// Setup datastore
    65  		failedBuild, _, analysis := testutil.CreateCompileFailureAnalysisAnalysisChain(
    66  			ctx, 88128398584903, "chromium", 444)
    67  		heuristicAnalysis := &model.CompileHeuristicAnalysis{
    68  			ParentAnalysis: datastore.KeyForObj(ctx, analysis),
    69  		}
    70  		So(datastore.Put(ctx, heuristicAnalysis), ShouldBeNil)
    71  
    72  		nsa := &model.CompileNthSectionAnalysis{
    73  			ParentAnalysis: datastore.KeyForObj(ctx, analysis),
    74  		}
    75  		So(datastore.Put(ctx, nsa), ShouldBeNil)
    76  		datastore.GetTestable(ctx).CatchupIndexes()
    77  
    78  		analysisURL := util.ConstructCompileAnalysisURL("chromium", failedBuild.Id)
    79  		buildURL := util.ConstructBuildURL(ctx, failedBuild.Id)
    80  		bugURL := util.ConstructBuganizerURLForAnalysis(analysisURL,
    81  			"https://test-review.googlesource.com/c/chromium/test/+/876543")
    82  
    83  		// Set up mock Gerrit client
    84  		ctl := gomock.NewController(t)
    85  		defer ctl.Finish()
    86  		mockClient := gerrit.NewMockedClient(ctx, ctl)
    87  		ctx = rotationproxy.MockedRotationProxyClientContext(mockClient.Ctx, map[string]string{
    88  			"oncallator:chrome-build-sheriff": `{"emails":["jdoe@example.com", "esmith@example.com"],"updated_unix_timestamp":1669331526}`,
    89  		})
    90  		// Set the project-level config for this test
    91  		gerritConfig := &configpb.GerritConfig{
    92  			ActionsEnabled: true,
    93  			CreateRevertSettings: &configpb.GerritConfig_RevertActionSettings{
    94  				Enabled:    true,
    95  				DailyLimit: 10,
    96  			},
    97  			SubmitRevertSettings: &configpb.GerritConfig_RevertActionSettings{
    98  				Enabled:    true,
    99  				DailyLimit: 4,
   100  			},
   101  			MaxRevertibleCulpritAge: 21600, // 6 hours
   102  			NthsectionSettings: &configpb.GerritConfig_NthSectionSettings{
   103  				Enabled:                     true,
   104  				ActionWhenVerificationError: false,
   105  			},
   106  		}
   107  		projectCfg := config.CreatePlaceholderProjectConfig()
   108  		projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
   109  		cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
   110  		So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
   111  
   112  		Convey("must be confirmed culprit", func() {
   113  			// Setup suspect in datastore
   114  			heuristicSuspect := &model.Suspect{
   115  				Id:             1,
   116  				Type:           model.SuspectType_Heuristic,
   117  				Score:          10,
   118  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   119  				GitilesCommit: buildbucketpb.GitilesCommit{
   120  					Host:    "test.googlesource.com",
   121  					Project: "chromium/src",
   122  					Id:      "12ab34cd56ef",
   123  				},
   124  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   125  				VerificationStatus: model.SuspectVerificationStatus_UnderVerification,
   126  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   127  			}
   128  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   129  			datastore.GetTestable(ctx).CatchupIndexes()
   130  
   131  			err := TakeCulpritAction(ctx, heuristicSuspect)
   132  			expectedErr := fmt.Sprintf("suspect (commit %s) has verification status"+
   133  				" %s and should not be reverted", heuristicSuspect.GitilesCommit.Id,
   134  				heuristicSuspect.VerificationStatus)
   135  			So(err, ShouldErrLike, expectedErr)
   136  
   137  			datastore.GetTestable(ctx).CatchupIndexes()
   138  			suspect, err := datastoreutil.GetSuspect(ctx,
   139  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   140  			So(err, ShouldBeNil)
   141  			So(suspect, ShouldNotBeNil)
   142  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   143  				RevertURL:               "",
   144  				IsRevertCreated:         false,
   145  				IsRevertCommitted:       false,
   146  				HasSupportRevertComment: false,
   147  				HasCulpritComment:       false,
   148  			})
   149  		})
   150  
   151  		Convey("nthsection actions must be enabled", func() {
   152  			// Set up suspect in datastore
   153  			nthsectionSuspect := &model.Suspect{
   154  				Type:           model.SuspectType_NthSection,
   155  				ParentAnalysis: datastore.KeyForObj(ctx, nsa),
   156  				GitilesCommit: buildbucketpb.GitilesCommit{
   157  					Host:    "test.googlesource.com",
   158  					Project: "chromium/src",
   159  					Id:      "12ab34cd56ef",
   160  				},
   161  				AnalysisType: pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   162  			}
   163  			So(datastore.Put(ctx, nthsectionSuspect), ShouldBeNil)
   164  			datastore.GetTestable(ctx).CatchupIndexes()
   165  
   166  			// Set the project-level config for this test
   167  			gerritConfig.NthsectionSettings.Enabled = false
   168  			projectCfg := config.CreatePlaceholderProjectConfig()
   169  			projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
   170  			cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
   171  			So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
   172  
   173  			err := TakeCulpritAction(ctx, nthsectionSuspect)
   174  			So(err, ShouldBeNil)
   175  
   176  			datastore.GetTestable(ctx).CatchupIndexes()
   177  			suspect, err := datastoreutil.GetSuspect(ctx,
   178  				nthsectionSuspect.Id, nthsectionSuspect.ParentAnalysis)
   179  			So(err, ShouldBeNil)
   180  			So(suspect, ShouldNotBeNil)
   181  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   182  				RevertURL:               "",
   183  				IsRevertCreated:         false,
   184  				IsRevertCommitted:       false,
   185  				HasSupportRevertComment: false,
   186  				HasCulpritComment:       false,
   187  			})
   188  		})
   189  
   190  		Convey("nthsection suspect must have correct status", func() {
   191  			// Set up suspect in datastore
   192  			nthsectionSuspect := &model.Suspect{
   193  				Type:           model.SuspectType_NthSection,
   194  				ParentAnalysis: datastore.KeyForObj(ctx, nsa),
   195  				GitilesCommit: buildbucketpb.GitilesCommit{
   196  					Host:    "test.googlesource.com",
   197  					Project: "chromium/src",
   198  					Id:      "12ab34cd56ef",
   199  				},
   200  				VerificationStatus: model.SuspectVerificationStatus_VerificationError,
   201  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   202  			}
   203  			So(datastore.Put(ctx, nthsectionSuspect), ShouldBeNil)
   204  			datastore.GetTestable(ctx).CatchupIndexes()
   205  
   206  			err := TakeCulpritAction(ctx, nthsectionSuspect)
   207  			expectedErr := fmt.Sprintf("suspect (commit %s) has verification status"+
   208  				" %s and should not be reverted", nthsectionSuspect.GitilesCommit.Id,
   209  				nthsectionSuspect.VerificationStatus)
   210  			So(err, ShouldErrLike, expectedErr)
   211  
   212  			datastore.GetTestable(ctx).CatchupIndexes()
   213  			suspect, err := datastoreutil.GetSuspect(ctx,
   214  				nthsectionSuspect.Id, nthsectionSuspect.ParentAnalysis)
   215  			So(err, ShouldBeNil)
   216  			So(suspect, ShouldNotBeNil)
   217  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   218  				RevertURL:               "",
   219  				IsRevertCreated:         false,
   220  				IsRevertCommitted:       false,
   221  				HasSupportRevertComment: false,
   222  				HasCulpritComment:       false,
   223  			})
   224  		})
   225  
   226  		Convey("all Gerrit actions disabled", func() {
   227  			// Setup suspect in datastore
   228  			heuristicSuspect := &model.Suspect{
   229  				Id:             2,
   230  				Type:           model.SuspectType_Heuristic,
   231  				Score:          10,
   232  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   233  				GitilesCommit: buildbucketpb.GitilesCommit{
   234  					Host:    "test.googlesource.com",
   235  					Project: "chromium/src",
   236  					Id:      "12ab34cd56ef",
   237  				},
   238  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   239  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   240  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   241  			}
   242  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   243  			datastore.GetTestable(ctx).CatchupIndexes()
   244  
   245  			// Set the project-level config for this test
   246  			gerritConfig.ActionsEnabled = false
   247  			projectCfg := config.CreatePlaceholderProjectConfig()
   248  			projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
   249  			cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
   250  			So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
   251  
   252  			err := TakeCulpritAction(ctx, heuristicSuspect)
   253  			So(err, ShouldBeNil)
   254  
   255  			datastore.GetTestable(ctx).CatchupIndexes()
   256  			suspect, err := datastoreutil.GetSuspect(ctx,
   257  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   258  			So(err, ShouldBeNil)
   259  			So(suspect, ShouldNotBeNil)
   260  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   261  				RevertURL:               "",
   262  				IsRevertCreated:         false,
   263  				IsRevertCommitted:       false,
   264  				HasSupportRevertComment: false,
   265  				HasCulpritComment:       false,
   266  				InactionReason:          pb.CulpritInactionReason_ACTIONS_DISABLED,
   267  			})
   268  		})
   269  
   270  		Convey("already reverted", func() {
   271  			// Setup suspect in datastore
   272  			heuristicSuspect := &model.Suspect{
   273  				Id:             3,
   274  				Type:           model.SuspectType_Heuristic,
   275  				Score:          10,
   276  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   277  				GitilesCommit: buildbucketpb.GitilesCommit{
   278  					Host:    "test.googlesource.com",
   279  					Project: "chromium/src",
   280  					Id:      "12ab34cd56ef",
   281  				},
   282  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   283  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   284  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   285  			}
   286  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   287  			datastore.GetTestable(ctx).CatchupIndexes()
   288  
   289  			// Set up mock responses
   290  			culpritRes := &gerritpb.ListChangesResponse{
   291  				Changes: []*gerritpb.ChangeInfo{{
   292  					Number:          876543,
   293  					Project:         "chromium/src",
   294  					Status:          gerritpb.ChangeStatus_MERGED,
   295  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   296  					CurrentRevision: "deadbeef",
   297  					Revisions: map[string]*gerritpb.RevisionInfo{
   298  						"deadbeef": {
   299  							Commit: &gerritpb.CommitInfo{
   300  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   301  								Author: &gerritpb.GitPersonInfo{
   302  									Name:  "John Doe",
   303  									Email: "jdoe@example.com",
   304  								},
   305  							},
   306  						},
   307  					},
   308  				}},
   309  			}
   310  			revertRes := &gerritpb.ListChangesResponse{
   311  				Changes: []*gerritpb.ChangeInfo{
   312  					{
   313  						Number:  876548,
   314  						Project: "chromium/src",
   315  						Status:  gerritpb.ChangeStatus_ABANDONED,
   316  					},
   317  					{
   318  						Number:  876549,
   319  						Project: "chromium/src",
   320  						Status:  gerritpb.ChangeStatus_MERGED,
   321  					},
   322  				},
   323  			}
   324  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   325  				Return(culpritRes, nil).Times(1)
   326  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   327  				Return(revertRes, nil).Times(1)
   328  
   329  			err := TakeCulpritAction(ctx, heuristicSuspect)
   330  			So(err, ShouldBeNil)
   331  
   332  			datastore.GetTestable(ctx).CatchupIndexes()
   333  			suspect, err := datastoreutil.GetSuspect(ctx,
   334  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   335  			So(err, ShouldBeNil)
   336  			So(suspect, ShouldNotBeNil)
   337  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   338  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
   339  				IsRevertCreated:         false,
   340  				IsRevertCommitted:       false,
   341  				HasSupportRevertComment: false,
   342  				HasCulpritComment:       false,
   343  				InactionReason:          pb.CulpritInactionReason_REVERTED_MANUALLY,
   344  			})
   345  		})
   346  
   347  		Convey("only abandoned revert exists", func() {
   348  			// Setup suspect in datastore
   349  			heuristicSuspect := &model.Suspect{
   350  				Id:             4,
   351  				Type:           model.SuspectType_Heuristic,
   352  				Score:          10,
   353  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   354  				GitilesCommit: buildbucketpb.GitilesCommit{
   355  					Host:    "test.googlesource.com",
   356  					Project: "chromium/src",
   357  					Id:      "12ab34cd56ef",
   358  				},
   359  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   360  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   361  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   362  			}
   363  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   364  			datastore.GetTestable(ctx).CatchupIndexes()
   365  
   366  			// Set up mock responses
   367  			culpritRes := &gerritpb.ListChangesResponse{
   368  				Changes: []*gerritpb.ChangeInfo{{
   369  					Number:          876543,
   370  					Project:         "chromium/src",
   371  					Status:          gerritpb.ChangeStatus_MERGED,
   372  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   373  					CurrentRevision: "deadbeef",
   374  					Revisions: map[string]*gerritpb.RevisionInfo{
   375  						"deadbeef": {
   376  							Commit: &gerritpb.CommitInfo{
   377  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   378  								Author: &gerritpb.GitPersonInfo{
   379  									Name:  "John Doe",
   380  									Email: "jdoe@example.com",
   381  								},
   382  							},
   383  						},
   384  					},
   385  				}},
   386  			}
   387  			revertRes := &gerritpb.ListChangesResponse{
   388  				Changes: []*gerritpb.ChangeInfo{{
   389  					Number:  876549,
   390  					Project: "chromium/src",
   391  					Status:  gerritpb.ChangeStatus_ABANDONED,
   392  				}},
   393  			}
   394  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   395  				Return(culpritRes, nil).Times(1)
   396  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   397  				Return(revertRes, nil).Times(1)
   398  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   399  				&gerritpb.SetReviewRequest{
   400  					Project:    culpritRes.Changes[0].Project,
   401  					Number:     culpritRes.Changes[0].Number,
   402  					RevisionId: "current",
   403  					Message: fmt.Sprintf("LUCI Bisection has identified this"+
   404  						" change as the culprit of a build failure. See the analysis: %s\n\n"+
   405  						"A revert for this change was not created because an abandoned"+
   406  						" revert already exists.\n\nSample failed build: %s\n\nIf this is"+
   407  						" a false positive, please report it at %s",
   408  						analysisURL, buildURL, bugURL),
   409  				},
   410  			)).Times(1)
   411  
   412  			err := TakeCulpritAction(ctx, heuristicSuspect)
   413  			So(err, ShouldBeNil)
   414  
   415  			datastore.GetTestable(ctx).CatchupIndexes()
   416  			suspect, err := datastoreutil.GetSuspect(ctx,
   417  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   418  			So(err, ShouldBeNil)
   419  			So(suspect, ShouldNotBeNil)
   420  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   421  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
   422  				IsRevertCreated:         false,
   423  				IsRevertCommitted:       false,
   424  				HasSupportRevertComment: false,
   425  				HasCulpritComment:       true,
   426  				CulpritCommentTime:      testclock.TestTimeUTC.Round(time.Second),
   427  			})
   428  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_culprit"), ShouldEqual, 1)
   429  		})
   430  
   431  		Convey("active revert exists", func() {
   432  			// Setup suspect in datastore
   433  			heuristicSuspect := &model.Suspect{
   434  				Id:             5,
   435  				Type:           model.SuspectType_Heuristic,
   436  				Score:          10,
   437  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   438  				GitilesCommit: buildbucketpb.GitilesCommit{
   439  					Host:    "test.googlesource.com",
   440  					Project: "chromium/src",
   441  					Id:      "12ab34cd56ef",
   442  				},
   443  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   444  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   445  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   446  			}
   447  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   448  			datastore.GetTestable(ctx).CatchupIndexes()
   449  
   450  			// Set up mock responses
   451  			culpritRes := &gerritpb.ListChangesResponse{
   452  				Changes: []*gerritpb.ChangeInfo{{
   453  					Number:          876543,
   454  					Project:         "chromium/src",
   455  					Status:          gerritpb.ChangeStatus_MERGED,
   456  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   457  					CurrentRevision: "deadbeef",
   458  					Revisions: map[string]*gerritpb.RevisionInfo{
   459  						"deadbeef": {
   460  							Commit: &gerritpb.CommitInfo{
   461  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   462  								Author: &gerritpb.GitPersonInfo{
   463  									Name:  "John Doe",
   464  									Email: "jdoe@example.com",
   465  								},
   466  							},
   467  						},
   468  					},
   469  				}},
   470  			}
   471  			revertRes := &gerritpb.ListChangesResponse{
   472  				Changes: []*gerritpb.ChangeInfo{
   473  					{
   474  						Number:  876548,
   475  						Project: "chromium/src",
   476  						Status:  gerritpb.ChangeStatus_ABANDONED,
   477  					},
   478  					{
   479  						Number:  876549,
   480  						Project: "chromium/src",
   481  						Status:  gerritpb.ChangeStatus_NEW,
   482  					},
   483  				},
   484  			}
   485  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   486  				Return(culpritRes, nil).Times(1)
   487  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   488  				Return(revertRes, nil).Times(1)
   489  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   490  				&gerritpb.SetReviewRequest{
   491  					Project:    revertRes.Changes[1].Project,
   492  					Number:     revertRes.Changes[1].Number,
   493  					RevisionId: "current",
   494  					Message: fmt.Sprintf("LUCI Bisection recommends submitting this"+
   495  						" revert because it has confirmed the target of this revert is the"+
   496  						" culprit of a build failure. See the analysis: %s\n\n"+
   497  						"Sample failed build: %s\n\nIf this is a false positive, please"+
   498  						" report it at %s", analysisURL, buildURL, bugURL),
   499  				},
   500  			)).Times(1)
   501  
   502  			err := TakeCulpritAction(ctx, heuristicSuspect)
   503  			So(err, ShouldBeNil)
   504  
   505  			datastore.GetTestable(ctx).CatchupIndexes()
   506  			suspect, err := datastoreutil.GetSuspect(ctx,
   507  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   508  			So(err, ShouldBeNil)
   509  			So(suspect, ShouldNotBeNil)
   510  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   511  				RevertURL:                "https://test-review.googlesource.com/c/chromium/src/+/876549",
   512  				IsRevertCreated:          false,
   513  				IsRevertCommitted:        false,
   514  				HasSupportRevertComment:  true,
   515  				SupportRevertCommentTime: testclock.TestTimeUTC.Round(time.Second),
   516  				HasCulpritComment:        false,
   517  			})
   518  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_revert"), ShouldEqual, 1)
   519  		})
   520  
   521  		Convey("non-sheriffable builder", func() {
   522  			failedBuild.SheriffRotations = []string{}
   523  			So(datastore.Put(ctx, failedBuild), ShouldBeNil)
   524  			datastore.GetTestable(ctx).CatchupIndexes()
   525  			// Setup suspect in datastore
   526  			heuristicSuspect := &model.Suspect{
   527  				Id:             6,
   528  				Type:           model.SuspectType_Heuristic,
   529  				Score:          10,
   530  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   531  				GitilesCommit: buildbucketpb.GitilesCommit{
   532  					Host:    "test.googlesource.com",
   533  					Project: "chromium/src",
   534  					Id:      "12ab34cd56ef",
   535  				},
   536  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   537  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   538  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   539  			}
   540  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   541  			datastore.GetTestable(ctx).CatchupIndexes()
   542  
   543  			// Set up mock responses
   544  			culpritRes := &gerritpb.ListChangesResponse{
   545  				Changes: []*gerritpb.ChangeInfo{{
   546  					Number:          876543,
   547  					Project:         "chromium/src",
   548  					Status:          gerritpb.ChangeStatus_MERGED,
   549  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   550  					CurrentRevision: "deadbeef",
   551  					Revisions: map[string]*gerritpb.RevisionInfo{
   552  						"deadbeef": {
   553  							Commit: &gerritpb.CommitInfo{
   554  								Message: "Title.\n\nBody is here.\n\nNOAUTOREVERT=true\n\nChange-Id: I100deadbeef",
   555  								Author: &gerritpb.GitPersonInfo{
   556  									Name:  "John Doe",
   557  									Email: "jdoe@example.com",
   558  								},
   559  							},
   560  						},
   561  					},
   562  				}},
   563  			}
   564  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   565  				Return(culpritRes, nil).Times(1)
   566  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   567  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
   568  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   569  				&gerritpb.SetReviewRequest{
   570  					Project:    culpritRes.Changes[0].Project,
   571  					Number:     culpritRes.Changes[0].Number,
   572  					RevisionId: "current",
   573  					Message: fmt.Sprintf("LUCI Bisection has identified this"+
   574  						" change as the culprit of a build failure. See the analysis: %s\n\n"+
   575  						"A revert for this change was not created because"+
   576  						" the associated builder is not being watched by gardeners.\n\n"+
   577  						"Sample failed build: %s\n\nIf this is a false positive, please report"+
   578  						" it at %s", analysisURL, buildURL, bugURL),
   579  				},
   580  			)).Times(1)
   581  
   582  			err := TakeCulpritAction(ctx, heuristicSuspect)
   583  			So(err, ShouldBeNil)
   584  
   585  			datastore.GetTestable(ctx).CatchupIndexes()
   586  			suspect, err := datastoreutil.GetSuspect(ctx,
   587  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   588  			So(err, ShouldBeNil)
   589  			So(suspect, ShouldNotBeNil)
   590  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   591  				RevertURL:               "",
   592  				IsRevertCreated:         false,
   593  				IsRevertCommitted:       false,
   594  				HasSupportRevertComment: false,
   595  				HasCulpritComment:       true,
   596  				CulpritCommentTime:      testclock.TestTimeUTC.Round(time.Second),
   597  			})
   598  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_culprit"), ShouldEqual, 1)
   599  		})
   600  
   601  		Convey("revert has auto-revert off flag set", func() {
   602  			// Setup suspect in datastore
   603  			heuristicSuspect := &model.Suspect{
   604  				Id:             6,
   605  				Type:           model.SuspectType_Heuristic,
   606  				Score:          10,
   607  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   608  				GitilesCommit: buildbucketpb.GitilesCommit{
   609  					Host:    "test.googlesource.com",
   610  					Project: "chromium/src",
   611  					Id:      "12ab34cd56ef",
   612  				},
   613  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   614  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   615  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   616  			}
   617  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   618  			datastore.GetTestable(ctx).CatchupIndexes()
   619  
   620  			// Set up mock responses
   621  			culpritRes := &gerritpb.ListChangesResponse{
   622  				Changes: []*gerritpb.ChangeInfo{{
   623  					Number:          876543,
   624  					Project:         "chromium/src",
   625  					Status:          gerritpb.ChangeStatus_MERGED,
   626  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   627  					CurrentRevision: "deadbeef",
   628  					Revisions: map[string]*gerritpb.RevisionInfo{
   629  						"deadbeef": {
   630  							Commit: &gerritpb.CommitInfo{
   631  								Message: "Title.\n\nBody is here.\n\nNOAUTOREVERT=true\n\nChange-Id: I100deadbeef",
   632  								Author: &gerritpb.GitPersonInfo{
   633  									Name:  "John Doe",
   634  									Email: "jdoe@example.com",
   635  								},
   636  							},
   637  						},
   638  					},
   639  				}},
   640  			}
   641  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   642  				Return(culpritRes, nil).Times(1)
   643  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   644  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
   645  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   646  				&gerritpb.SetReviewRequest{
   647  					Project:    culpritRes.Changes[0].Project,
   648  					Number:     culpritRes.Changes[0].Number,
   649  					RevisionId: "current",
   650  					Message: fmt.Sprintf("LUCI Bisection has identified this"+
   651  						" change as the culprit of a build failure. See the analysis: %s\n\n"+
   652  						"A revert for this change was not created because"+
   653  						" auto-revert has been disabled for this CL by its description.\n\n"+
   654  						"Sample failed build: %s\n\nIf this is a false positive, please report"+
   655  						" it at %s", analysisURL, buildURL, bugURL),
   656  				},
   657  			)).Times(1)
   658  
   659  			err := TakeCulpritAction(ctx, heuristicSuspect)
   660  			So(err, ShouldBeNil)
   661  
   662  			datastore.GetTestable(ctx).CatchupIndexes()
   663  			suspect, err := datastoreutil.GetSuspect(ctx,
   664  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   665  			So(err, ShouldBeNil)
   666  			So(suspect, ShouldNotBeNil)
   667  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   668  				RevertURL:               "",
   669  				IsRevertCreated:         false,
   670  				IsRevertCommitted:       false,
   671  				HasSupportRevertComment: false,
   672  				HasCulpritComment:       true,
   673  				CulpritCommentTime:      testclock.TestTimeUTC.Round(time.Second),
   674  			})
   675  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_culprit"), ShouldEqual, 1)
   676  		})
   677  
   678  		Convey("revert was from an irrevertible author", func() {
   679  			// Setup suspect in datastore
   680  			heuristicSuspect := &model.Suspect{
   681  				Id:             7,
   682  				Type:           model.SuspectType_Heuristic,
   683  				Score:          10,
   684  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   685  				GitilesCommit: buildbucketpb.GitilesCommit{
   686  					Host:    "test.googlesource.com",
   687  					Project: "chromium/src",
   688  					Id:      "12ab34cd56ef",
   689  				},
   690  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   691  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   692  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   693  			}
   694  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   695  			datastore.GetTestable(ctx).CatchupIndexes()
   696  
   697  			// Set up mock responses
   698  			culpritRes := &gerritpb.ListChangesResponse{
   699  				Changes: []*gerritpb.ChangeInfo{{
   700  					Number:          876543,
   701  					Project:         "chromium/src",
   702  					Status:          gerritpb.ChangeStatus_MERGED,
   703  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   704  					CurrentRevision: "deadbeef",
   705  					Revisions: map[string]*gerritpb.RevisionInfo{
   706  						"deadbeef": {
   707  							Commit: &gerritpb.CommitInfo{
   708  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   709  								Author: &gerritpb.GitPersonInfo{
   710  									Name:  "ChromeOS Commit Bot",
   711  									Email: "chromeos-commit-bot@chromium.org",
   712  								},
   713  							},
   714  						},
   715  					},
   716  				}},
   717  			}
   718  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   719  				Return(culpritRes, nil).Times(1)
   720  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   721  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
   722  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   723  				&gerritpb.SetReviewRequest{
   724  					Project:    culpritRes.Changes[0].Project,
   725  					Number:     culpritRes.Changes[0].Number,
   726  					RevisionId: "current",
   727  					Message: fmt.Sprintf("LUCI Bisection has identified this"+
   728  						" change as the culprit of a build failure. See the analysis: %s\n\n"+
   729  						"A revert for this change was not created because"+
   730  						" LUCI Bisection cannot revert changes from this CL's author.\n\n"+
   731  						"Sample failed build: %s\n\nIf this is a false positive, please report"+
   732  						" it at %s", analysisURL, buildURL, bugURL),
   733  				},
   734  			)).Times(1)
   735  
   736  			err := TakeCulpritAction(ctx, heuristicSuspect)
   737  			So(err, ShouldBeNil)
   738  
   739  			datastore.GetTestable(ctx).CatchupIndexes()
   740  			suspect, err := datastoreutil.GetSuspect(ctx,
   741  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   742  			So(err, ShouldBeNil)
   743  			So(suspect, ShouldNotBeNil)
   744  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   745  				RevertURL:               "",
   746  				IsRevertCreated:         false,
   747  				IsRevertCommitted:       false,
   748  				HasSupportRevertComment: false,
   749  				HasCulpritComment:       true,
   750  				CulpritCommentTime:      testclock.TestTimeUTC.Round(time.Second),
   751  			})
   752  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_culprit"), ShouldEqual, 1)
   753  		})
   754  
   755  		Convey("culprit has a downstream dependency", func() {
   756  			// Setup suspect in datastore
   757  			heuristicSuspect := &model.Suspect{
   758  				Id:             8,
   759  				Type:           model.SuspectType_Heuristic,
   760  				Score:          10,
   761  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   762  				GitilesCommit: buildbucketpb.GitilesCommit{
   763  					Host:    "test.googlesource.com",
   764  					Project: "chromium/src",
   765  					Id:      "12ab34cd56ef",
   766  				},
   767  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   768  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   769  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   770  			}
   771  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   772  			datastore.GetTestable(ctx).CatchupIndexes()
   773  
   774  			// Set up mock responses
   775  			culpritRes := &gerritpb.ListChangesResponse{
   776  				Changes: []*gerritpb.ChangeInfo{{
   777  					Number:          876543,
   778  					Project:         "chromium/src",
   779  					Status:          gerritpb.ChangeStatus_MERGED,
   780  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   781  					CurrentRevision: "deadbeef",
   782  					Revisions: map[string]*gerritpb.RevisionInfo{
   783  						"deadbeef": {
   784  							Commit: &gerritpb.CommitInfo{
   785  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   786  								Author: &gerritpb.GitPersonInfo{
   787  									Name:  "John Doe",
   788  									Email: "jdoe@example.com",
   789  								},
   790  							},
   791  						},
   792  					},
   793  				}},
   794  			}
   795  			revertRes := &gerritpb.ListChangesResponse{
   796  				Changes: []*gerritpb.ChangeInfo{},
   797  			}
   798  			relatedChanges := &gerritpb.GetRelatedChangesResponse{
   799  				Changes: []*gerritpb.GetRelatedChangesResponse_ChangeAndCommit{
   800  					{
   801  						Project: "chromium/src",
   802  						Number:  876544,
   803  						Status:  gerritpb.ChangeStatus_MERGED,
   804  					},
   805  					{
   806  						Project: "chromium/src",
   807  						Number:  876543,
   808  						Status:  gerritpb.ChangeStatus_MERGED,
   809  					},
   810  				},
   811  			}
   812  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   813  				Return(culpritRes, nil).Times(1)
   814  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   815  				Return(revertRes, nil).Times(1)
   816  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
   817  				Return(relatedChanges, nil).Times(1)
   818  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   819  				&gerritpb.SetReviewRequest{
   820  					Project:    culpritRes.Changes[0].Project,
   821  					Number:     culpritRes.Changes[0].Number,
   822  					RevisionId: "current",
   823  					Message: fmt.Sprintf("LUCI Bisection has identified this"+
   824  						" change as the culprit of a build failure. See the analysis: %s\n\n"+
   825  						"A revert for this change was not created because there are merged"+
   826  						" changes depending on it.\n\nSample failed build: %s\n\nIf this is"+
   827  						" a false positive, please report it at %s",
   828  						analysisURL, buildURL, bugURL),
   829  				},
   830  			)).Times(1)
   831  
   832  			err := TakeCulpritAction(ctx, heuristicSuspect)
   833  			So(err, ShouldBeNil)
   834  
   835  			datastore.GetTestable(ctx).CatchupIndexes()
   836  			suspect, err := datastoreutil.GetSuspect(ctx,
   837  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   838  			So(err, ShouldBeNil)
   839  			So(suspect, ShouldNotBeNil)
   840  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   841  				RevertURL:               "",
   842  				IsRevertCreated:         false,
   843  				IsRevertCommitted:       false,
   844  				HasSupportRevertComment: false,
   845  				HasCulpritComment:       true,
   846  				CulpritCommentTime:      testclock.TestTimeUTC.Round(time.Second),
   847  			})
   848  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_culprit"), ShouldEqual, 1)
   849  		})
   850  
   851  		Convey("revert creation is disabled", func() {
   852  			// Setup suspect in datastore
   853  			heuristicSuspect := &model.Suspect{
   854  				Id:             9,
   855  				Type:           model.SuspectType_Heuristic,
   856  				Score:          10,
   857  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   858  				GitilesCommit: buildbucketpb.GitilesCommit{
   859  					Host:    "test.googlesource.com",
   860  					Project: "chromium/src",
   861  					Id:      "12ab34cd56ef",
   862  				},
   863  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   864  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   865  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   866  			}
   867  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   868  			datastore.GetTestable(ctx).CatchupIndexes()
   869  
   870  			// Set the project-level config for this test
   871  			gerritConfig.CreateRevertSettings.Enabled = false
   872  			projectCfg := config.CreatePlaceholderProjectConfig()
   873  			projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
   874  			cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
   875  			So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
   876  
   877  			// Set up mock responses
   878  			culpritRes := &gerritpb.ListChangesResponse{
   879  				Changes: []*gerritpb.ChangeInfo{{
   880  					Number:          876543,
   881  					Project:         "chromium/src",
   882  					Status:          gerritpb.ChangeStatus_MERGED,
   883  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
   884  					CurrentRevision: "deadbeef",
   885  					Revisions: map[string]*gerritpb.RevisionInfo{
   886  						"deadbeef": {
   887  							Commit: &gerritpb.CommitInfo{
   888  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   889  								Author: &gerritpb.GitPersonInfo{
   890  									Name:  "John Doe",
   891  									Email: "jdoe@example.com",
   892  								},
   893  							},
   894  						},
   895  					},
   896  				}},
   897  			}
   898  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   899  				Return(culpritRes, nil).Times(1)
   900  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   901  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
   902  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
   903  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
   904  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   905  				&gerritpb.SetReviewRequest{
   906  					Project:    culpritRes.Changes[0].Project,
   907  					Number:     culpritRes.Changes[0].Number,
   908  					RevisionId: "current",
   909  					Message: fmt.Sprintf("LUCI Bisection has identified this"+
   910  						" change as the culprit of a build failure. See the analysis: %s\n\n"+
   911  						"A revert for this change was not created because"+
   912  						" LUCI Bisection's revert creation has been disabled.\n\n"+
   913  						"Sample failed build: %s\n\nIf this is a false positive, please"+
   914  						" report it at %s", analysisURL, buildURL, bugURL),
   915  				},
   916  			)).Times(1)
   917  
   918  			err := TakeCulpritAction(ctx, heuristicSuspect)
   919  			So(err, ShouldBeNil)
   920  
   921  			datastore.GetTestable(ctx).CatchupIndexes()
   922  			suspect, err := datastoreutil.GetSuspect(ctx,
   923  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
   924  			So(err, ShouldBeNil)
   925  			So(suspect, ShouldNotBeNil)
   926  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
   927  				RevertURL:               "",
   928  				IsRevertCreated:         false,
   929  				IsRevertCommitted:       false,
   930  				HasSupportRevertComment: false,
   931  				HasCulpritComment:       true,
   932  				CulpritCommentTime:      testclock.TestTimeUTC.Round(time.Second),
   933  			})
   934  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "comment_culprit"), ShouldEqual, 1)
   935  		})
   936  
   937  		Convey("culprit was committed too long ago", func() {
   938  			// Setup suspect in datastore
   939  			heuristicSuspect := &model.Suspect{
   940  				Id:             10,
   941  				Type:           model.SuspectType_Heuristic,
   942  				Score:          10,
   943  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
   944  				GitilesCommit: buildbucketpb.GitilesCommit{
   945  					Host:    "test.googlesource.com",
   946  					Project: "chromium/src",
   947  					Id:      "12ab34cd56ef",
   948  				},
   949  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
   950  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
   951  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
   952  			}
   953  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
   954  			datastore.GetTestable(ctx).CatchupIndexes()
   955  
   956  			// Set up mock responses
   957  			culpritRes := &gerritpb.ListChangesResponse{
   958  				Changes: []*gerritpb.ChangeInfo{{
   959  					Number:          876543,
   960  					Project:         "chromium/src",
   961  					Status:          gerritpb.ChangeStatus_MERGED,
   962  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 30)),
   963  					CurrentRevision: "deadbeef",
   964  					Revisions: map[string]*gerritpb.RevisionInfo{
   965  						"deadbeef": {
   966  							Commit: &gerritpb.CommitInfo{
   967  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
   968  								Author: &gerritpb.GitPersonInfo{
   969  									Name:  "John Doe",
   970  									Email: "jdoe@example.com",
   971  								},
   972  							},
   973  						},
   974  					},
   975  				}},
   976  			}
   977  			revertRes := &gerritpb.ChangeInfo{
   978  				Number:  876549,
   979  				Project: "chromium/src",
   980  				Status:  gerritpb.ChangeStatus_NEW,
   981  			}
   982  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   983  				Return(culpritRes, nil).Times(1)
   984  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   985  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
   986  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
   987  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
   988  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
   989  				Return(revertRes, nil).Times(1)
   990  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
   991  				Return(&gerritpb.ListChangesResponse{
   992  					Changes: []*gerritpb.ChangeInfo{revertRes},
   993  				}, nil).Times(1)
   994  			mockClient.Client.EXPECT().GetChange(gomock.Any(), gomock.Any()).
   995  				Return(revertRes, nil).Times(1)
   996  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
   997  				&gerritpb.SetReviewRequest{
   998  					Project:    revertRes.Project,
   999  					Number:     revertRes.Number,
  1000  					RevisionId: "current",
  1001  					Message: "LUCI Bisection could not automatically submit this revert" +
  1002  						" because the target of this revert was not committed recently.",
  1003  					Reviewers: []*gerritpb.ReviewerInput{
  1004  						{
  1005  							Reviewer: "jdoe@example.com",
  1006  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_REVIEWER,
  1007  						},
  1008  						{
  1009  							Reviewer: "esmith@example.com",
  1010  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_REVIEWER,
  1011  						},
  1012  					},
  1013  				},
  1014  			)).Times(1)
  1015  
  1016  			err := TakeCulpritAction(ctx, heuristicSuspect)
  1017  			So(err, ShouldBeNil)
  1018  
  1019  			datastore.GetTestable(ctx).CatchupIndexes()
  1020  			suspect, err := datastoreutil.GetSuspect(ctx,
  1021  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1022  			So(err, ShouldBeNil)
  1023  			So(suspect, ShouldNotBeNil)
  1024  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1025  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1026  				IsRevertCreated:         true,
  1027  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1028  				IsRevertCommitted:       false,
  1029  				HasSupportRevertComment: false,
  1030  				HasCulpritComment:       false,
  1031  			})
  1032  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1033  		})
  1034  
  1035  		Convey("revert commit is disabled", func() {
  1036  			// Setup suspect in datastore
  1037  			heuristicSuspect := &model.Suspect{
  1038  				Id:             11,
  1039  				Type:           model.SuspectType_Heuristic,
  1040  				Score:          10,
  1041  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
  1042  				GitilesCommit: buildbucketpb.GitilesCommit{
  1043  					Host:    "test.googlesource.com",
  1044  					Project: "chromium/src",
  1045  					Id:      "12ab34cd56ef",
  1046  				},
  1047  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1048  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1049  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1050  			}
  1051  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
  1052  			datastore.GetTestable(ctx).CatchupIndexes()
  1053  
  1054  			// Set the project-level config for this test
  1055  			gerritConfig.SubmitRevertSettings.Enabled = false
  1056  			projectCfg := config.CreatePlaceholderProjectConfig()
  1057  			projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
  1058  			cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
  1059  			So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
  1060  
  1061  			// Set up mock responses
  1062  			culpritRes := &gerritpb.ListChangesResponse{
  1063  				Changes: []*gerritpb.ChangeInfo{{
  1064  					Number:          876543,
  1065  					Project:         "chromium/src",
  1066  					Status:          gerritpb.ChangeStatus_MERGED,
  1067  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1068  					CurrentRevision: "deadbeef",
  1069  					Revisions: map[string]*gerritpb.RevisionInfo{
  1070  						"deadbeef": {
  1071  							Commit: &gerritpb.CommitInfo{
  1072  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1073  								Author: &gerritpb.GitPersonInfo{
  1074  									Name:  "John Doe",
  1075  									Email: "jdoe@example.com",
  1076  								},
  1077  							},
  1078  						},
  1079  					},
  1080  				}},
  1081  			}
  1082  			revertRes := &gerritpb.ChangeInfo{
  1083  				Number:  876549,
  1084  				Project: "chromium/src",
  1085  				Status:  gerritpb.ChangeStatus_NEW,
  1086  			}
  1087  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1088  				Return(culpritRes, nil).Times(1)
  1089  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1090  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1091  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1092  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1093  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1094  				Return(revertRes, nil).Times(1)
  1095  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1096  				Return(&gerritpb.ListChangesResponse{
  1097  					Changes: []*gerritpb.ChangeInfo{revertRes},
  1098  				}, nil).Times(1)
  1099  			mockClient.Client.EXPECT().GetChange(gomock.Any(), gomock.Any()).
  1100  				Return(revertRes, nil).Times(1)
  1101  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
  1102  				&gerritpb.SetReviewRequest{
  1103  					Project:    revertRes.Project,
  1104  					Number:     revertRes.Number,
  1105  					RevisionId: "current",
  1106  					Message: "LUCI Bisection could not automatically submit this revert" +
  1107  						" because LUCI Bisection's revert submission has been disabled.",
  1108  					Reviewers: []*gerritpb.ReviewerInput{
  1109  						{
  1110  							Reviewer: "jdoe@example.com",
  1111  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_REVIEWER,
  1112  						},
  1113  						{
  1114  							Reviewer: "esmith@example.com",
  1115  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_REVIEWER,
  1116  						},
  1117  					},
  1118  				},
  1119  			)).Times(1)
  1120  
  1121  			err := TakeCulpritAction(ctx, heuristicSuspect)
  1122  			So(err, ShouldBeNil)
  1123  
  1124  			datastore.GetTestable(ctx).CatchupIndexes()
  1125  			suspect, err := datastoreutil.GetSuspect(ctx,
  1126  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1127  			So(err, ShouldBeNil)
  1128  			So(suspect, ShouldNotBeNil)
  1129  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1130  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1131  				IsRevertCreated:         true,
  1132  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1133  				IsRevertCommitted:       false,
  1134  				HasSupportRevertComment: false,
  1135  				HasCulpritComment:       false,
  1136  			})
  1137  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1138  		})
  1139  
  1140  		Convey("revert for culprit is created and bot-committed", func() {
  1141  			// Setup suspect in datastore
  1142  			heuristicSuspect := &model.Suspect{
  1143  				Id:             12,
  1144  				Type:           model.SuspectType_Heuristic,
  1145  				Score:          10,
  1146  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
  1147  				GitilesCommit: buildbucketpb.GitilesCommit{
  1148  					Host:    "test.googlesource.com",
  1149  					Project: "chromium/src",
  1150  					Id:      "12ab34cd56ef",
  1151  				},
  1152  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1153  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1154  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1155  			}
  1156  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
  1157  			datastore.GetTestable(ctx).CatchupIndexes()
  1158  
  1159  			// Set up mock responses
  1160  			culpritRes := &gerritpb.ListChangesResponse{
  1161  				Changes: []*gerritpb.ChangeInfo{{
  1162  					Number:          876543,
  1163  					Project:         "chromium/src",
  1164  					Status:          gerritpb.ChangeStatus_MERGED,
  1165  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1166  					CurrentRevision: "deadbeef",
  1167  					Revisions: map[string]*gerritpb.RevisionInfo{
  1168  						"deadbeef": {
  1169  							Commit: &gerritpb.CommitInfo{
  1170  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1171  								Author: &gerritpb.GitPersonInfo{
  1172  									Name:  "John Doe",
  1173  									Email: "jdoe@example.com",
  1174  								},
  1175  							},
  1176  						},
  1177  					},
  1178  				}},
  1179  			}
  1180  			revertRes := &gerritpb.ChangeInfo{
  1181  				Number:  876549,
  1182  				Project: "chromium/src",
  1183  				Status:  gerritpb.ChangeStatus_NEW,
  1184  			}
  1185  			pureRevertRes := &gerritpb.PureRevertInfo{
  1186  				IsPureRevert: true,
  1187  			}
  1188  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1189  				Return(culpritRes, nil).Times(1)
  1190  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1191  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1192  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1193  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1194  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1195  				Return(revertRes, nil).Times(1)
  1196  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1197  				Return(&gerritpb.ListChangesResponse{
  1198  					Changes: []*gerritpb.ChangeInfo{revertRes},
  1199  				}, nil).Times(1)
  1200  			mockClient.Client.EXPECT().GetChange(gomock.Any(), gomock.Any()).
  1201  				Return(revertRes, nil).Times(1)
  1202  			mockClient.Client.EXPECT().GetPureRevert(gomock.Any(), gomock.Any()).
  1203  				Return(pureRevertRes, nil).Times(1)
  1204  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
  1205  				&gerritpb.SetReviewRequest{
  1206  					Project:    revertRes.Project,
  1207  					Number:     revertRes.Number,
  1208  					RevisionId: "current",
  1209  					Message:    "LUCI Bisection is automatically submitting this revert.",
  1210  					Labels: map[string]int32{
  1211  						"Owners-Override": 1,
  1212  						"Bot-Commit":      1,
  1213  						"Commit-Queue":    2,
  1214  					},
  1215  					Reviewers: []*gerritpb.ReviewerInput{
  1216  						{
  1217  							Reviewer: "jdoe@example.com",
  1218  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_CC,
  1219  						},
  1220  						{
  1221  							Reviewer: "esmith@example.com",
  1222  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_CC,
  1223  						},
  1224  					},
  1225  				},
  1226  			)).Times(1)
  1227  
  1228  			err := TakeCulpritAction(ctx, heuristicSuspect)
  1229  			So(err, ShouldBeNil)
  1230  
  1231  			datastore.GetTestable(ctx).CatchupIndexes()
  1232  			suspect, err := datastoreutil.GetSuspect(ctx,
  1233  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1234  			So(err, ShouldBeNil)
  1235  			So(suspect, ShouldNotBeNil)
  1236  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1237  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1238  				IsRevertCreated:         true,
  1239  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1240  				IsRevertCommitted:       true,
  1241  				RevertCommitTime:        testclock.TestTimeUTC.Round(time.Second),
  1242  				HasSupportRevertComment: false,
  1243  				HasCulpritComment:       false,
  1244  			})
  1245  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1246  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "submit_revert"), ShouldEqual, 1)
  1247  		})
  1248  
  1249  		Convey("revert for culprit is created then manually committed", func() {
  1250  			// Setup suspect in datastore
  1251  			heuristicSuspect := &model.Suspect{
  1252  				Id:             13,
  1253  				Type:           model.SuspectType_Heuristic,
  1254  				Score:          10,
  1255  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
  1256  				GitilesCommit: buildbucketpb.GitilesCommit{
  1257  					Host:    "test.googlesource.com",
  1258  					Project: "chromium/src",
  1259  					Id:      "12ab34cd56ef",
  1260  				},
  1261  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1262  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1263  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1264  			}
  1265  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
  1266  			datastore.GetTestable(ctx).CatchupIndexes()
  1267  
  1268  			// Set up mock responses
  1269  			culpritRes := &gerritpb.ListChangesResponse{
  1270  				Changes: []*gerritpb.ChangeInfo{{
  1271  					Number:          876543,
  1272  					Project:         "chromium/src",
  1273  					Status:          gerritpb.ChangeStatus_MERGED,
  1274  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1275  					CurrentRevision: "deadbeef",
  1276  					Revisions: map[string]*gerritpb.RevisionInfo{
  1277  						"deadbeef": {
  1278  							Commit: &gerritpb.CommitInfo{
  1279  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1280  								Author: &gerritpb.GitPersonInfo{
  1281  									Name:  "John Doe",
  1282  									Email: "jdoe@example.com",
  1283  								},
  1284  							},
  1285  						},
  1286  					},
  1287  				}},
  1288  			}
  1289  			revertRes := &gerritpb.ChangeInfo{
  1290  				Number:  876549,
  1291  				Project: "chromium/src",
  1292  				Status:  gerritpb.ChangeStatus_NEW,
  1293  			}
  1294  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1295  				Return(culpritRes, nil).Times(1)
  1296  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1297  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1298  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1299  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1300  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1301  				Return(revertRes, nil).Times(1)
  1302  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1303  				Return(&gerritpb.ListChangesResponse{
  1304  					Changes: []*gerritpb.ChangeInfo{
  1305  						{
  1306  							Number:  876549,
  1307  							Project: "chromium/src",
  1308  							Status:  gerritpb.ChangeStatus_MERGED,
  1309  						},
  1310  					},
  1311  				}, nil).Times(1)
  1312  
  1313  			err := TakeCulpritAction(ctx, heuristicSuspect)
  1314  			So(err, ShouldBeNil)
  1315  
  1316  			datastore.GetTestable(ctx).CatchupIndexes()
  1317  			suspect, err := datastoreutil.GetSuspect(ctx,
  1318  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1319  			So(err, ShouldBeNil)
  1320  			So(suspect, ShouldNotBeNil)
  1321  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1322  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1323  				IsRevertCreated:         true,
  1324  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1325  				IsRevertCommitted:       false,
  1326  				HasSupportRevertComment: false,
  1327  				HasCulpritComment:       false,
  1328  			})
  1329  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1330  		})
  1331  
  1332  		Convey("revert for culprit is created but another revert was merged in the meantime", func() {
  1333  			// Setup suspect in datastore
  1334  			heuristicSuspect := &model.Suspect{
  1335  				Id:             14,
  1336  				Type:           model.SuspectType_Heuristic,
  1337  				Score:          10,
  1338  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
  1339  				GitilesCommit: buildbucketpb.GitilesCommit{
  1340  					Host:    "test.googlesource.com",
  1341  					Project: "chromium/src",
  1342  					Id:      "12ab34cd56ef",
  1343  				},
  1344  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1345  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1346  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1347  			}
  1348  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
  1349  			datastore.GetTestable(ctx).CatchupIndexes()
  1350  
  1351  			// Set up mock responses
  1352  			culpritRes := &gerritpb.ListChangesResponse{
  1353  				Changes: []*gerritpb.ChangeInfo{{
  1354  					Number:          876543,
  1355  					Project:         "chromium/src",
  1356  					Status:          gerritpb.ChangeStatus_MERGED,
  1357  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1358  					CurrentRevision: "deadbeef",
  1359  					Revisions: map[string]*gerritpb.RevisionInfo{
  1360  						"deadbeef": {
  1361  							Commit: &gerritpb.CommitInfo{
  1362  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1363  								Author: &gerritpb.GitPersonInfo{
  1364  									Name:  "John Doe",
  1365  									Email: "jdoe@example.com",
  1366  								},
  1367  							},
  1368  						},
  1369  					},
  1370  				}},
  1371  			}
  1372  			revertRes := &gerritpb.ChangeInfo{
  1373  				Number:  876549,
  1374  				Project: "chromium/src",
  1375  				Status:  gerritpb.ChangeStatus_NEW,
  1376  			}
  1377  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1378  				Return(culpritRes, nil).Times(1)
  1379  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1380  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1381  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1382  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1383  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1384  				Return(revertRes, nil).Times(1)
  1385  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1386  				Return(&gerritpb.ListChangesResponse{
  1387  					Changes: []*gerritpb.ChangeInfo{
  1388  						{
  1389  							Number:  876549,
  1390  							Project: "chromium/src",
  1391  							Status:  gerritpb.ChangeStatus_NEW,
  1392  						},
  1393  						{
  1394  							Number:  876551,
  1395  							Project: "chromium/src",
  1396  							Status:  gerritpb.ChangeStatus_MERGED,
  1397  						},
  1398  					},
  1399  				}, nil).Times(1)
  1400  
  1401  			err := TakeCulpritAction(ctx, heuristicSuspect)
  1402  			So(err, ShouldBeNil)
  1403  
  1404  			datastore.GetTestable(ctx).CatchupIndexes()
  1405  			suspect, err := datastoreutil.GetSuspect(ctx,
  1406  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1407  			So(err, ShouldBeNil)
  1408  			So(suspect, ShouldNotBeNil)
  1409  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1410  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1411  				IsRevertCreated:         true,
  1412  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1413  				IsRevertCommitted:       false,
  1414  				HasSupportRevertComment: false,
  1415  				HasCulpritComment:       false,
  1416  			})
  1417  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1418  		})
  1419  
  1420  		Convey("revert can be created and bot-committed even if creation request times out", func() {
  1421  			// Setup suspect in datastore
  1422  			heuristicSuspect := &model.Suspect{
  1423  				Id:             15,
  1424  				Type:           model.SuspectType_Heuristic,
  1425  				Score:          10,
  1426  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
  1427  				GitilesCommit: buildbucketpb.GitilesCommit{
  1428  					Host:    "test.googlesource.com",
  1429  					Project: "chromium/src",
  1430  					Id:      "12ab34cd56ef",
  1431  				},
  1432  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1433  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1434  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1435  			}
  1436  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
  1437  			datastore.GetTestable(ctx).CatchupIndexes()
  1438  
  1439  			// Set up mock responses
  1440  			culpritRes := &gerritpb.ListChangesResponse{
  1441  				Changes: []*gerritpb.ChangeInfo{{
  1442  					Number:          876543,
  1443  					Project:         "chromium/src",
  1444  					Status:          gerritpb.ChangeStatus_MERGED,
  1445  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1446  					Subject:         "Title.",
  1447  					CurrentRevision: "deadbeef",
  1448  					Revisions: map[string]*gerritpb.RevisionInfo{
  1449  						"deadbeef": {
  1450  							Commit: &gerritpb.CommitInfo{
  1451  								Message: `Title.
  1452  
  1453  Body is here.
  1454  
  1455  Change-Id: I100deadbeef`,
  1456  								Author: &gerritpb.GitPersonInfo{
  1457  									Name:  "John Doe",
  1458  									Email: "jdoe@example.com",
  1459  								},
  1460  							},
  1461  						},
  1462  					},
  1463  				}},
  1464  			}
  1465  			lbEmail, err := gerrit.ServiceAccountEmail(ctx)
  1466  			So(err, ShouldBeNil)
  1467  			revertRes := &gerritpb.ChangeInfo{
  1468  				Number:  876549,
  1469  				Project: "chromium/src",
  1470  				Status:  gerritpb.ChangeStatus_NEW,
  1471  				Owner: &gerritpb.AccountInfo{
  1472  					Email: lbEmail,
  1473  				},
  1474  				CurrentRevision: "deadbeff",
  1475  				Revisions: map[string]*gerritpb.RevisionInfo{
  1476  					"deadbeff": {
  1477  						Commit: &gerritpb.CommitInfo{
  1478  							Message: fmt.Sprintf(
  1479  								`Revert "Title."
  1480  
  1481  This reverts commit 12ab34cd56ef.
  1482  
  1483  Reason for revert:
  1484  LUCI Bisection has identified this change as the culprit of a build failure. See the analysis: %s
  1485  
  1486  Sample failed build: %s
  1487  
  1488  If this is a false positive, please report it at %s
  1489  
  1490  Original change's description:
  1491  > Title.
  1492  >
  1493  > Body is here.
  1494  >
  1495  > Change-Id: I100deadbeef
  1496  
  1497  Change-Id: 987654321abcdef
  1498  No-Presubmit: true
  1499  No-Tree-Checks: true
  1500  No-Try: true`, analysisURL, buildURL, bugURL),
  1501  						},
  1502  					},
  1503  				},
  1504  			}
  1505  			pureRevertRes := &gerritpb.PureRevertInfo{
  1506  				IsPureRevert: true,
  1507  			}
  1508  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1509  				Return(culpritRes, nil).Times(1)
  1510  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1511  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1512  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1513  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1514  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1515  				Return(nil, status.Errorf(codes.DeadlineExceeded, "revert creation timed out")).
  1516  				Times(1)
  1517  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1518  				Return(&gerritpb.ListChangesResponse{
  1519  					Changes: []*gerritpb.ChangeInfo{revertRes},
  1520  				}, nil).Times(2)
  1521  			mockClient.Client.EXPECT().GetChange(gomock.Any(), gomock.Any()).
  1522  				Return(revertRes, nil).Times(1)
  1523  			mockClient.Client.EXPECT().GetPureRevert(gomock.Any(), gomock.Any()).
  1524  				Return(pureRevertRes, nil).Times(1)
  1525  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
  1526  				&gerritpb.SetReviewRequest{
  1527  					Project:    revertRes.Project,
  1528  					Number:     revertRes.Number,
  1529  					RevisionId: "current",
  1530  					Message:    "LUCI Bisection is automatically submitting this revert.",
  1531  					Labels: map[string]int32{
  1532  						"Owners-Override": 1,
  1533  						"Bot-Commit":      1,
  1534  						"Commit-Queue":    2,
  1535  					},
  1536  					Reviewers: []*gerritpb.ReviewerInput{
  1537  						{
  1538  							Reviewer: "jdoe@example.com",
  1539  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_CC,
  1540  						},
  1541  						{
  1542  							Reviewer: "esmith@example.com",
  1543  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_CC,
  1544  						},
  1545  					},
  1546  				},
  1547  			)).Times(1)
  1548  
  1549  			err = TakeCulpritAction(ctx, heuristicSuspect)
  1550  			So(err, ShouldBeNil)
  1551  
  1552  			datastore.GetTestable(ctx).CatchupIndexes()
  1553  			suspect, err := datastoreutil.GetSuspect(ctx,
  1554  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1555  			So(err, ShouldBeNil)
  1556  			So(suspect, ShouldNotBeNil)
  1557  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1558  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1559  				IsRevertCreated:         true,
  1560  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1561  				IsRevertCommitted:       true,
  1562  				RevertCommitTime:        testclock.TestTimeUTC.Round(time.Second),
  1563  				HasSupportRevertComment: false,
  1564  				HasCulpritComment:       false,
  1565  			})
  1566  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1567  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "submit_revert"), ShouldEqual, 1)
  1568  		})
  1569  
  1570  		Convey("revert is not bot-committed for non-timeout error when creating a revert", func() {
  1571  			// Setup suspect in datastore
  1572  			heuristicSuspect := &model.Suspect{
  1573  				Id:             16,
  1574  				Type:           model.SuspectType_Heuristic,
  1575  				Score:          10,
  1576  				ParentAnalysis: datastore.KeyForObj(ctx, heuristicAnalysis),
  1577  				GitilesCommit: buildbucketpb.GitilesCommit{
  1578  					Host:    "test.googlesource.com",
  1579  					Project: "chromium/src",
  1580  					Id:      "12ab34cd56ef",
  1581  				},
  1582  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1583  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1584  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1585  			}
  1586  			So(datastore.Put(ctx, heuristicSuspect), ShouldBeNil)
  1587  			datastore.GetTestable(ctx).CatchupIndexes()
  1588  
  1589  			// Set up mock responses
  1590  			culpritRes := &gerritpb.ListChangesResponse{
  1591  				Changes: []*gerritpb.ChangeInfo{{
  1592  					Number:          876543,
  1593  					Project:         "chromium/src",
  1594  					Status:          gerritpb.ChangeStatus_MERGED,
  1595  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1596  					Subject:         "Title.",
  1597  					CurrentRevision: "deadbeef",
  1598  					Revisions: map[string]*gerritpb.RevisionInfo{
  1599  						"deadbeef": {
  1600  							Commit: &gerritpb.CommitInfo{
  1601  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1602  								Author: &gerritpb.GitPersonInfo{
  1603  									Name:  "John Doe",
  1604  									Email: "jdoe@example.com",
  1605  								},
  1606  							},
  1607  						},
  1608  					},
  1609  				}},
  1610  			}
  1611  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1612  				Return(culpritRes, nil).Times(1)
  1613  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1614  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1615  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1616  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1617  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1618  				Return(nil, status.Errorf(codes.Internal, "revert creation failed internally")).
  1619  				Times(1)
  1620  
  1621  			err := TakeCulpritAction(ctx, heuristicSuspect)
  1622  			So(err, ShouldNotBeNil)
  1623  
  1624  			datastore.GetTestable(ctx).CatchupIndexes()
  1625  			suspect, err := datastoreutil.GetSuspect(ctx,
  1626  				heuristicSuspect.Id, heuristicSuspect.ParentAnalysis)
  1627  			So(err, ShouldBeNil)
  1628  			So(suspect, ShouldNotBeNil)
  1629  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1630  				RevertURL:               "",
  1631  				IsRevertCreated:         false,
  1632  				IsRevertCommitted:       false,
  1633  				HasSupportRevertComment: false,
  1634  				HasCulpritComment:       false,
  1635  			})
  1636  		})
  1637  
  1638  		Convey("revert for nthsection suspect is created although verification error", func() {
  1639  			// Setup suspect in datastore
  1640  			suspect := &model.Suspect{
  1641  				Id:                 16,
  1642  				ParentAnalysis:     datastore.KeyForObj(ctx, nsa),
  1643  				VerificationStatus: model.SuspectVerificationStatus_VerificationError,
  1644  				Type:               model.SuspectType_NthSection,
  1645  				GitilesCommit: buildbucketpb.GitilesCommit{
  1646  					Host:    "test.googlesource.com",
  1647  					Project: "chromium/src",
  1648  					Id:      "12ab34cd56ef",
  1649  				},
  1650  				ReviewUrl:    "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1651  				AnalysisType: pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1652  			}
  1653  			So(datastore.Put(ctx, suspect), ShouldBeNil)
  1654  			datastore.GetTestable(ctx).CatchupIndexes()
  1655  
  1656  			gerritConfig.NthsectionSettings.ActionWhenVerificationError = true
  1657  			projectCfg := config.CreatePlaceholderProjectConfig()
  1658  			projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
  1659  			cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
  1660  			So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
  1661  
  1662  			// Set up mock responses
  1663  			culpritRes := &gerritpb.ListChangesResponse{
  1664  				Changes: []*gerritpb.ChangeInfo{{
  1665  					Number:          876543,
  1666  					Project:         "chromium/src",
  1667  					Status:          gerritpb.ChangeStatus_MERGED,
  1668  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1669  					CurrentRevision: "deadbeef",
  1670  					Revisions: map[string]*gerritpb.RevisionInfo{
  1671  						"deadbeef": {
  1672  							Commit: &gerritpb.CommitInfo{
  1673  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1674  								Author: &gerritpb.GitPersonInfo{
  1675  									Name:  "John Doe",
  1676  									Email: "jdoe@example.com",
  1677  								},
  1678  							},
  1679  						},
  1680  					},
  1681  				}},
  1682  			}
  1683  			revertRes := &gerritpb.ChangeInfo{
  1684  				Number:  876549,
  1685  				Project: "chromium/src",
  1686  				Status:  gerritpb.ChangeStatus_NEW,
  1687  			}
  1688  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1689  				Return(culpritRes, nil).Times(1)
  1690  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1691  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1692  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1693  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1694  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1695  				Return(revertRes, nil).Times(1)
  1696  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1697  				Return(&gerritpb.ListChangesResponse{
  1698  					Changes: []*gerritpb.ChangeInfo{
  1699  						{
  1700  							Number:  876549,
  1701  							Project: "chromium/src",
  1702  							Status:  gerritpb.ChangeStatus_MERGED,
  1703  						},
  1704  					},
  1705  				}, nil).Times(1)
  1706  
  1707  			err := TakeCulpritAction(ctx, suspect)
  1708  			So(err, ShouldBeNil)
  1709  
  1710  			datastore.GetTestable(ctx).CatchupIndexes()
  1711  			suspect, err = datastoreutil.GetSuspect(ctx,
  1712  				suspect.Id, suspect.ParentAnalysis)
  1713  			So(err, ShouldBeNil)
  1714  			So(suspect, ShouldNotBeNil)
  1715  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1716  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1717  				IsRevertCreated:         true,
  1718  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1719  				IsRevertCommitted:       false,
  1720  				HasSupportRevertComment: false,
  1721  				HasCulpritComment:       false,
  1722  			})
  1723  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1724  		})
  1725  
  1726  		Convey("revert for culprit is created and bot-committed for nthsection", func() {
  1727  			// Setup suspect in datastore
  1728  			suspect := &model.Suspect{
  1729  				Id:             14,
  1730  				Type:           model.SuspectType_NthSection,
  1731  				ParentAnalysis: datastore.KeyForObj(ctx, nsa),
  1732  				GitilesCommit: buildbucketpb.GitilesCommit{
  1733  					Host:    "test.googlesource.com",
  1734  					Project: "chromium/src",
  1735  					Id:      "12ab34cd56ef",
  1736  				},
  1737  				ReviewUrl:          "https://test-review.googlesource.com/c/chromium/test/+/876543",
  1738  				VerificationStatus: model.SuspectVerificationStatus_ConfirmedCulprit,
  1739  				AnalysisType:       pb.AnalysisType_COMPILE_FAILURE_ANALYSIS,
  1740  			}
  1741  			So(datastore.Put(ctx, suspect), ShouldBeNil)
  1742  			datastore.GetTestable(ctx).CatchupIndexes()
  1743  
  1744  			// Set the project-level config for this test
  1745  			gerritConfig.NthsectionSettings.ActionWhenVerificationError = true
  1746  			projectCfg := config.CreatePlaceholderProjectConfig()
  1747  			projectCfg.CompileAnalysisConfig.GerritConfig = gerritConfig
  1748  			cfg := map[string]*configpb.ProjectConfig{"chromium": projectCfg}
  1749  			So(config.SetTestProjectConfig(ctx, cfg), ShouldBeNil)
  1750  
  1751  			// Set up mock responses
  1752  			culpritRes := &gerritpb.ListChangesResponse{
  1753  				Changes: []*gerritpb.ChangeInfo{{
  1754  					Number:          876543,
  1755  					Project:         "chromium/src",
  1756  					Status:          gerritpb.ChangeStatus_MERGED,
  1757  					Submitted:       timestamppb.New(clock.Now(ctx).Add(-time.Hour * 3)),
  1758  					CurrentRevision: "deadbeef",
  1759  					Revisions: map[string]*gerritpb.RevisionInfo{
  1760  						"deadbeef": {
  1761  							Commit: &gerritpb.CommitInfo{
  1762  								Message: "Title.\n\nBody is here.\n\nChange-Id: I100deadbeef",
  1763  								Author: &gerritpb.GitPersonInfo{
  1764  									Name:  "John Doe",
  1765  									Email: "jdoe@example.com",
  1766  								},
  1767  							},
  1768  						},
  1769  					},
  1770  				}},
  1771  			}
  1772  			revertRes := &gerritpb.ChangeInfo{
  1773  				Number:  876549,
  1774  				Project: "chromium/src",
  1775  				Status:  gerritpb.ChangeStatus_NEW,
  1776  			}
  1777  			pureRevertRes := &gerritpb.PureRevertInfo{
  1778  				IsPureRevert: true,
  1779  			}
  1780  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1781  				Return(culpritRes, nil).Times(1)
  1782  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1783  				Return(&gerritpb.ListChangesResponse{}, nil).Times(1)
  1784  			mockClient.Client.EXPECT().GetRelatedChanges(gomock.Any(), gomock.Any()).
  1785  				Return(&gerritpb.GetRelatedChangesResponse{}, nil).Times(1)
  1786  			mockClient.Client.EXPECT().RevertChange(gomock.Any(), gomock.Any()).
  1787  				Return(revertRes, nil).Times(1)
  1788  			mockClient.Client.EXPECT().ListChanges(gomock.Any(), gomock.Any()).
  1789  				Return(&gerritpb.ListChangesResponse{
  1790  					Changes: []*gerritpb.ChangeInfo{revertRes},
  1791  				}, nil).Times(1)
  1792  			mockClient.Client.EXPECT().GetChange(gomock.Any(), gomock.Any()).
  1793  				Return(revertRes, nil).Times(1)
  1794  			mockClient.Client.EXPECT().GetPureRevert(gomock.Any(), gomock.Any()).
  1795  				Return(pureRevertRes, nil).Times(1)
  1796  			mockClient.Client.EXPECT().SetReview(gomock.Any(), proto.MatcherEqual(
  1797  				&gerritpb.SetReviewRequest{
  1798  					Project:    revertRes.Project,
  1799  					Number:     revertRes.Number,
  1800  					RevisionId: "current",
  1801  					Message:    "LUCI Bisection is automatically submitting this revert.",
  1802  					Labels: map[string]int32{
  1803  						"Owners-Override": 1,
  1804  						"Bot-Commit":      1,
  1805  						"Commit-Queue":    2,
  1806  					},
  1807  					Reviewers: []*gerritpb.ReviewerInput{
  1808  						{
  1809  							Reviewer: "jdoe@example.com",
  1810  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_CC,
  1811  						},
  1812  						{
  1813  							Reviewer: "esmith@example.com",
  1814  							State:    gerritpb.ReviewerInput_REVIEWER_INPUT_STATE_CC,
  1815  						},
  1816  					},
  1817  				},
  1818  			)).Times(1)
  1819  
  1820  			err := TakeCulpritAction(ctx, suspect)
  1821  			So(err, ShouldBeNil)
  1822  
  1823  			datastore.GetTestable(ctx).CatchupIndexes()
  1824  			suspect, err = datastoreutil.GetSuspect(ctx,
  1825  				suspect.Id, suspect.ParentAnalysis)
  1826  			So(err, ShouldBeNil)
  1827  			So(suspect, ShouldNotBeNil)
  1828  			So(suspect.ActionDetails, ShouldResemble, model.ActionDetails{
  1829  				RevertURL:               "https://test-review.googlesource.com/c/chromium/src/+/876549",
  1830  				IsRevertCreated:         true,
  1831  				RevertCreateTime:        testclock.TestTimeUTC.Round(time.Second),
  1832  				IsRevertCommitted:       true,
  1833  				RevertCommitTime:        testclock.TestTimeUTC.Round(time.Second),
  1834  				HasSupportRevertComment: false,
  1835  				HasCulpritComment:       false,
  1836  			})
  1837  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "create_revert"), ShouldEqual, 1)
  1838  			So(culpritActionCounter.Get(ctx, "chromium", "compile", "submit_revert"), ShouldEqual, 1)
  1839  		})
  1840  
  1841  	})
  1842  }