go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/internal/changepoints/bqexporter/bqexporter_test.go (about) 1 // Copyright 2023 The LUCI Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package bqexporter 16 17 import ( 18 "context" 19 "encoding/hex" 20 "testing" 21 "time" 22 23 "google.golang.org/protobuf/types/known/timestamppb" 24 25 "go.chromium.org/luci/analysis/internal/changepoints/inputbuffer" 26 cpb "go.chromium.org/luci/analysis/internal/changepoints/proto" 27 "go.chromium.org/luci/analysis/internal/changepoints/testvariantbranch" 28 bqpb "go.chromium.org/luci/analysis/proto/bq" 29 pb "go.chromium.org/luci/analysis/proto/v1" 30 31 . "github.com/smartystreets/goconvey/convey" 32 . "go.chromium.org/luci/common/testing/assertions" 33 ) 34 35 func TestBQExporter(t *testing.T) { 36 Convey(`Export test variant branches`, t, func() { 37 ctx := context.Background() 38 client := NewFakeClient() 39 exporter := NewExporter(client) 40 41 // Create row input data. 42 variant := &pb.Variant{ 43 Def: map[string]string{ 44 "k": "v", 45 }, 46 } 47 48 sourceRef := &pb.SourceRef{ 49 System: &pb.SourceRef_Gitiles{ 50 Gitiles: &pb.GitilesRef{ 51 Host: "host", 52 Project: "proj", 53 Ref: "ref", 54 }, 55 }, 56 } 57 58 type RowInput struct { 59 TestVariantBranch *testvariantbranch.Entry 60 InputBufferSegments []*inputbuffer.Segment 61 } 62 63 row1 := &RowInput{ 64 TestVariantBranch: &testvariantbranch.Entry{ 65 Project: "chromium", 66 TestID: "test_id_1", 67 VariantHash: "variant_hash_1", 68 RefHash: []byte("refhash1"), 69 Variant: variant, 70 SourceRef: sourceRef, 71 InputBuffer: &inputbuffer.Buffer{}, 72 }, 73 InputBufferSegments: []*inputbuffer.Segment{ 74 { 75 HasStartChangepoint: false, 76 StartPosition: 1, 77 StartPositionLowerBound99Th: 1, 78 StartPositionUpperBound99Th: 3, 79 EndPosition: 6, 80 StartHour: timestamppb.New(time.Unix(3600, 0)), 81 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 82 Counts: &cpb.Counts{ 83 TotalResults: 9, 84 }, 85 }, 86 }, 87 } 88 89 row2 := &RowInput{ 90 TestVariantBranch: &testvariantbranch.Entry{ 91 Project: "chromium", 92 TestID: "test_id_2", 93 VariantHash: "variant_hash_2", 94 RefHash: []byte("refhash2"), 95 Variant: variant, 96 SourceRef: sourceRef, 97 InputBuffer: &inputbuffer.Buffer{}, 98 FinalizedSegments: &cpb.Segments{ 99 Segments: []*cpb.Segment{ 100 { 101 State: cpb.SegmentState_FINALIZED, 102 HasStartChangepoint: false, 103 StartPosition: 1, 104 StartHour: timestamppb.New(time.Unix(7000*3600, 0)), 105 EndPosition: 10, 106 EndHour: timestamppb.New(time.Unix(8000*3600, 0)), 107 // Less than 90 days ago 108 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(9000*3600, 0)), 109 FinalizedCounts: &cpb.Counts{ 110 TotalVerdicts: 3, 111 UnexpectedVerdicts: 2, 112 FlakyVerdicts: 1, 113 TotalRuns: 7, 114 UnexpectedUnretriedRuns: 6, 115 UnexpectedAfterRetryRuns: 5, 116 FlakyRuns: 4, 117 TotalResults: 9, 118 UnexpectedResults: 8, 119 }, 120 }, 121 { 122 State: cpb.SegmentState_FINALIZED, 123 HasStartChangepoint: true, 124 StartPosition: 11, 125 StartHour: timestamppb.New(time.Unix(7000*3600, 0)), 126 StartPositionLowerBound_99Th: 9, 127 StartPositionUpperBound_99Th: 13, 128 EndPosition: 20, 129 EndHour: timestamppb.New(time.Unix(8000*3600, 0)), 130 FinalizedCounts: &cpb.Counts{ 131 TotalVerdicts: 5, 132 }, 133 }, 134 }, 135 }, 136 FinalizingSegment: &cpb.Segment{ 137 State: cpb.SegmentState_FINALIZING, 138 HasStartChangepoint: true, 139 StartPosition: 21, 140 EndPosition: 30, 141 StartHour: timestamppb.New(time.Unix(7000*3600, 0)), 142 StartPositionLowerBound_99Th: 19, 143 StartPositionUpperBound_99Th: 23, 144 EndHour: timestamppb.New(time.Unix(8000*3600, 0)), 145 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(9000*3600, 0)), 146 FinalizedCounts: &cpb.Counts{ 147 TotalVerdicts: 4, 148 UnexpectedVerdicts: 3, 149 FlakyVerdicts: 1, 150 TotalRuns: 6, 151 UnexpectedUnretriedRuns: 3, 152 UnexpectedAfterRetryRuns: 2, 153 FlakyRuns: 5, 154 TotalResults: 10, 155 UnexpectedResults: 9, 156 }, 157 }, 158 }, 159 InputBufferSegments: []*inputbuffer.Segment{ 160 { 161 HasStartChangepoint: false, 162 StartPosition: 31, 163 StartPositionLowerBound99Th: 29, 164 StartPositionUpperBound99Th: 33, 165 EndPosition: 40, 166 StartHour: timestamppb.New(time.Unix(3600, 0)), 167 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 168 MostRecentUnexpectedResultHourAllVerdicts: timestamppb.New(time.Unix(9000*3600, 0)), 169 Counts: &cpb.Counts{ 170 TotalVerdicts: 5, 171 UnexpectedVerdicts: 1, 172 FlakyVerdicts: 1, 173 TotalRuns: 7, 174 UnexpectedUnretriedRuns: 2, 175 UnexpectedAfterRetryRuns: 3, 176 FlakyRuns: 1, 177 TotalResults: 11, 178 UnexpectedResults: 6, 179 }, 180 }, 181 { 182 HasStartChangepoint: true, 183 StartPosition: 41, 184 StartPositionLowerBound99Th: 39, 185 StartPositionUpperBound99Th: 43, 186 EndPosition: 40, 187 StartHour: timestamppb.New(time.Unix(3600, 0)), 188 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 189 Counts: &cpb.Counts{ 190 TotalVerdicts: 6, 191 }, 192 }, 193 }, 194 } 195 196 row3 := &RowInput{ 197 TestVariantBranch: &testvariantbranch.Entry{ 198 Project: "chromium", 199 TestID: "test_id_3", 200 VariantHash: "variant_hash_3", 201 RefHash: []byte("refhash3"), 202 Variant: variant, 203 SourceRef: sourceRef, 204 InputBuffer: &inputbuffer.Buffer{}, 205 }, 206 InputBufferSegments: []*inputbuffer.Segment{ 207 { 208 HasStartChangepoint: true, 209 StartPosition: 1, 210 StartPositionLowerBound99Th: 1, 211 StartPositionUpperBound99Th: 3, 212 EndPosition: 6, 213 StartHour: timestamppb.New(time.Unix(3600, 0)), 214 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 215 // More than 90 days ago. 216 MostRecentUnexpectedResultHourAllVerdicts: timestamppb.New(time.Unix(7000*3600, 0)), 217 Counts: &cpb.Counts{ 218 TotalResults: 9, 219 UnexpectedResults: 4, 220 }, 221 }, 222 }, 223 } 224 225 var bqRows []PartialBigQueryRow 226 for _, row := range []*RowInput{row1, row2, row3} { 227 bqRow, err := ToPartialBigQueryRow(row.TestVariantBranch, row.InputBufferSegments) 228 So(err, ShouldBeNil) 229 bqRows = append(bqRows, bqRow) 230 } 231 232 ris := RowInputs{ 233 Rows: bqRows, 234 CommitTimestamp: time.Unix(10000*3600, 0), 235 } 236 err := exporter.ExportTestVariantBranches(ctx, ris) 237 So(err, ShouldBeNil) 238 rows := client.Insertions 239 So(len(rows), ShouldEqual, 3) 240 241 // Asserts the rows. 242 So(rows[0], ShouldResembleProto, &bqpb.TestVariantBranchRow{ 243 Project: "chromium", 244 TestId: "test_id_1", 245 VariantHash: "variant_hash_1", 246 RefHash: hex.EncodeToString([]byte("refhash1")), 247 Variant: "{\"k\":\"v\"}", 248 Ref: sourceRef, 249 Version: timestamppb.New(time.Unix(10000*3600, 0)), 250 Segments: []*bqpb.Segment{ 251 { 252 HasStartChangepoint: false, 253 StartPosition: 1, 254 StartPositionLowerBound_99Th: 1, 255 StartPositionUpperBound_99Th: 3, 256 EndPosition: 6, 257 StartHour: timestamppb.New(time.Unix(3600, 0)), 258 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 259 Counts: &bqpb.Segment_Counts{ 260 TotalResults: 9, 261 }, 262 }, 263 }, 264 }) 265 266 So(rows[1], ShouldResembleProto, &bqpb.TestVariantBranchRow{ 267 Project: "chromium", 268 TestId: "test_id_2", 269 VariantHash: "variant_hash_2", 270 RefHash: hex.EncodeToString([]byte("refhash2")), 271 Variant: "{\"k\":\"v\"}", 272 Ref: sourceRef, 273 Version: timestamppb.New(time.Unix(10000*3600, 0)), 274 HasRecentUnexpectedResults: 1, 275 Segments: []*bqpb.Segment{ 276 { 277 HasStartChangepoint: true, 278 StartPosition: 41, 279 StartPositionLowerBound_99Th: 39, 280 StartPositionUpperBound_99Th: 43, 281 EndPosition: 40, 282 StartHour: timestamppb.New(time.Unix(3600, 0)), 283 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 284 Counts: &bqpb.Segment_Counts{ 285 TotalVerdicts: 6, 286 }, 287 }, 288 { 289 HasStartChangepoint: true, 290 StartPosition: 21, 291 StartPositionLowerBound_99Th: 19, 292 StartPositionUpperBound_99Th: 23, 293 StartHour: timestamppb.New(time.Unix(7000*3600, 0)), 294 EndPosition: 40, 295 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 296 Counts: &bqpb.Segment_Counts{ 297 TotalVerdicts: 9, 298 UnexpectedVerdicts: 4, 299 FlakyVerdicts: 2, 300 TotalRuns: 13, 301 UnexpectedUnretriedRuns: 5, 302 UnexpectedAfterRetryRuns: 5, 303 FlakyRuns: 6, 304 TotalResults: 21, 305 UnexpectedResults: 15, 306 }, 307 }, 308 { 309 HasStartChangepoint: true, 310 StartPosition: 11, 311 StartHour: timestamppb.New(time.Unix(7000*3600, 0)), 312 StartPositionLowerBound_99Th: 9, 313 StartPositionUpperBound_99Th: 13, 314 EndPosition: 20, 315 EndHour: timestamppb.New(time.Unix(8000*3600, 0)), 316 Counts: &bqpb.Segment_Counts{ 317 TotalVerdicts: 5, 318 }, 319 }, 320 { 321 HasStartChangepoint: false, 322 StartPosition: 1, 323 StartHour: timestamppb.New(time.Unix(7000*3600, 0)), 324 EndPosition: 10, 325 EndHour: timestamppb.New(time.Unix(8000*3600, 0)), 326 Counts: &bqpb.Segment_Counts{ 327 TotalVerdicts: 3, 328 UnexpectedVerdicts: 2, 329 FlakyVerdicts: 1, 330 TotalRuns: 7, 331 UnexpectedUnretriedRuns: 6, 332 UnexpectedAfterRetryRuns: 5, 333 FlakyRuns: 4, 334 TotalResults: 9, 335 UnexpectedResults: 8, 336 }, 337 }, 338 }, 339 }) 340 341 So(rows[2], ShouldResembleProto, &bqpb.TestVariantBranchRow{ 342 Project: "chromium", 343 TestId: "test_id_3", 344 VariantHash: "variant_hash_3", 345 RefHash: hex.EncodeToString([]byte("refhash3")), 346 Variant: "{\"k\":\"v\"}", 347 Ref: sourceRef, 348 Version: timestamppb.New(time.Unix(10000*3600, 0)), 349 Segments: []*bqpb.Segment{ 350 { 351 HasStartChangepoint: true, 352 StartPosition: 1, 353 StartPositionLowerBound_99Th: 1, 354 StartPositionUpperBound_99Th: 3, 355 EndPosition: 6, 356 StartHour: timestamppb.New(time.Unix(3600, 0)), 357 EndHour: timestamppb.New(time.Unix(6*3600, 0)), 358 Counts: &bqpb.Segment_Counts{ 359 TotalResults: 9, 360 UnexpectedResults: 4, 361 }, 362 }, 363 }, 364 }) 365 }) 366 }