go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/cv/internal/run/bq/exporter_test.go (about)

     1  // Copyright 2021 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bq
    16  
    17  import (
    18  	"context"
    19  	"testing"
    20  	"time"
    21  
    22  	"google.golang.org/protobuf/reflect/protoreflect"
    23  	"google.golang.org/protobuf/types/known/timestamppb"
    24  
    25  	gerritpb "go.chromium.org/luci/common/proto/gerrit"
    26  	"go.chromium.org/luci/gae/service/datastore"
    27  	"go.chromium.org/luci/server/tq/tqtesting"
    28  
    29  	cvbqpb "go.chromium.org/luci/cv/api/bigquery/v1"
    30  	"go.chromium.org/luci/cv/internal/changelist"
    31  	"go.chromium.org/luci/cv/internal/common"
    32  	"go.chromium.org/luci/cv/internal/cvtesting"
    33  	"go.chromium.org/luci/cv/internal/run"
    34  
    35  	. "github.com/smartystreets/goconvey/convey"
    36  	. "go.chromium.org/luci/common/testing/assertions"
    37  )
    38  
    39  func TestExportRunToBQ(t *testing.T) {
    40  	t.Parallel()
    41  
    42  	Convey("Exporting a Run to BQ works", t, func() {
    43  		ct := cvtesting.Test{}
    44  		ctx, cancel := ct.SetUp(t)
    45  		defer cancel()
    46  
    47  		exporter := NewExporter(ct.TQDispatcher, ct.BQFake, ct.Env)
    48  
    49  		// Set up datastore by putting a sample Run + RunCLs.
    50  		epoch := ct.Clock.Now().UTC()
    51  		runID := common.MakeRunID("lproject", epoch, 1, []byte("aaa"))
    52  		r := run.Run{
    53  			ID:            runID,
    54  			Status:        run.Status_SUCCEEDED,
    55  			ConfigGroupID: "sha256:deadbeefdeadbeef/cgroup",
    56  			CreateTime:    epoch,
    57  			StartTime:     epoch.Add(time.Minute * 2),
    58  			EndTime:       epoch.Add(time.Minute * 25),
    59  			CLs:           common.CLIDs{1},
    60  			Submission:    nil,
    61  			Mode:          run.DryRun,
    62  		}
    63  		So(datastore.Put(ctx, &r), ShouldBeNil)
    64  		So(datastore.Put(ctx, &run.RunCL{
    65  			ID:         1,
    66  			Run:        datastore.MakeKey(ctx, common.RunKind, string(runID)),
    67  			ExternalID: "gerrit/foo-review.googlesource.com/111",
    68  			Detail: &changelist.Snapshot{
    69  				LuciProject:           "lproject",
    70  				Patchset:              2,
    71  				MinEquivalentPatchset: 2,
    72  				Kind: &changelist.Snapshot_Gerrit{
    73  					Gerrit: &changelist.Gerrit{
    74  						Host: "foo-review.googlesource.com",
    75  						Info: &gerritpb.ChangeInfo{
    76  							Number:  111,
    77  							Project: "gproject",
    78  							Ref:     "refs/heads/main",
    79  						},
    80  					},
    81  				},
    82  			},
    83  			Trigger: &run.Trigger{Time: timestamppb.New(epoch)},
    84  		}), ShouldBeNil)
    85  		So(nil, ShouldBeNil)
    86  
    87  		// BQ Export tasks must be scheduled in a transaction.
    88  		schedule := func() error {
    89  			return datastore.RunInTransaction(ctx, func(tCtx context.Context) error {
    90  				return exporter.Schedule(tCtx, runID)
    91  			}, nil)
    92  		}
    93  
    94  		Convey("A row is sent", func() {
    95  			Convey("in prod", func() {
    96  				So(schedule(), ShouldBeNil)
    97  				ct.TQ.Run(ctx, tqtesting.StopAfterTask(exportRunToBQTaskClass))
    98  				rows := ct.BQFake.Rows("", CVDataset, CVTable)
    99  				So(rows, ShouldResembleProto, []protoreflect.ProtoMessage{&cvbqpb.Attempt{
   100  					Key:                  "616161",
   101  					LuciProject:          "lproject",
   102  					ConfigGroup:          "cgroup",
   103  					ClGroupKey:           "331ea2a6a5d5f3b3",
   104  					EquivalentClGroupKey: "47337d4707144297",
   105  					StartTime:            timestamppb.New(epoch),
   106  					ActualStartTime:      timestamppb.New(epoch.Add(time.Minute * 2)),
   107  					EndTime:              timestamppb.New(epoch.Add(25 * time.Minute)),
   108  					GerritChanges: []*cvbqpb.GerritChange{
   109  						{
   110  							Host:                       "foo-review.googlesource.com",
   111  							Project:                    "gproject",
   112  							Change:                     111,
   113  							Patchset:                   2,
   114  							EarliestEquivalentPatchset: 2,
   115  							TriggerTime:                timestamppb.New(epoch),
   116  							Mode:                       cvbqpb.Mode_DRY_RUN,
   117  							SubmitStatus:               cvbqpb.GerritChange_PENDING,
   118  						},
   119  					},
   120  					Status:    cvbqpb.AttemptStatus_SUCCESS,
   121  					Substatus: cvbqpb.AttemptSubstatus_NO_SUBSTATUS,
   122  				}})
   123  
   124  				// The same rows must be sent to legacy CQ table.
   125  				cqRows := ct.BQFake.Rows(legacyProject, legacyDataset, legacyTable)
   126  				So(cqRows, ShouldResembleProto, rows)
   127  				// And only these 2 rows have been sent.
   128  				So(ct.BQFake.TotalSent(), ShouldEqual, 2)
   129  			})
   130  
   131  			Convey("in dev", func() {
   132  				So(schedule(), ShouldBeNil)
   133  				ct.Env.IsGAEDev = true
   134  				ct.TQ.Run(ctx, tqtesting.StopAfterTask(exportRunToBQTaskClass))
   135  				So(ct.BQFake.Rows("", CVDataset, CVTable), ShouldHaveLength, 1)
   136  				// Must not send to production legacy.
   137  				So(ct.BQFake.Rows(legacyProject, legacyDataset, legacyTable), ShouldHaveLength, 0)
   138  				So(ct.BQFake.Rows(legacyProjectDev, legacyDataset, legacyTable), ShouldHaveLength, 1)
   139  			})
   140  		})
   141  	})
   142  }