go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/internal/changepoints/testvariantbranch/test_variant_branch_test.go (about) 1 // Copyright 2023 The LUCI Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package testvariantbranch 16 17 import ( 18 "testing" 19 "time" 20 21 "google.golang.org/protobuf/types/known/timestamppb" 22 23 rdbpb "go.chromium.org/luci/resultdb/proto/v1" 24 "go.chromium.org/luci/server/span" 25 26 "go.chromium.org/luci/analysis/internal/changepoints/inputbuffer" 27 cpb "go.chromium.org/luci/analysis/internal/changepoints/proto" 28 tu "go.chromium.org/luci/analysis/internal/changepoints/testutil" 29 "go.chromium.org/luci/analysis/internal/testutil" 30 pb "go.chromium.org/luci/analysis/proto/v1" 31 32 . "github.com/smartystreets/goconvey/convey" 33 . "go.chromium.org/luci/common/testing/assertions" 34 ) 35 36 func TestFetchUpdateTestVariantBranch(t *testing.T) { 37 Convey("Fetch not found", t, func() { 38 ctx := testutil.IntegrationTestContext(t) 39 key := Key{ 40 Project: "proj", 41 TestID: "test_id", 42 VariantHash: "variant_hash", 43 RefHash: "git_hash", 44 } 45 tvbs, err := Read(span.Single(ctx), []Key{key}) 46 So(err, ShouldBeNil) 47 So(len(tvbs), ShouldEqual, 1) 48 So(tvbs[0], ShouldBeNil) 49 }) 50 51 Convey("Insert and fetch", t, func() { 52 ctx := testutil.IntegrationTestContext(t) 53 tvb1 := &Entry{ 54 IsNew: true, 55 Project: "proj_1", 56 TestID: "test_id_1", 57 VariantHash: "variant_hash_1", 58 RefHash: []byte("refhash1"), 59 Variant: &pb.Variant{ 60 Def: map[string]string{ 61 "key1": "val1", 62 "key2": "val2", 63 }, 64 }, 65 SourceRef: &pb.SourceRef{ 66 System: &pb.SourceRef_Gitiles{ 67 Gitiles: &pb.GitilesRef{ 68 Host: "host_1", 69 Project: "proj_1", 70 Ref: "ref_1", 71 }, 72 }, 73 }, 74 InputBuffer: &inputbuffer.Buffer{ 75 HotBufferCapacity: 100, 76 HotBuffer: inputbuffer.History{ 77 Verdicts: []inputbuffer.PositionVerdict{ 78 { 79 CommitPosition: 15, 80 IsSimpleExpectedPass: true, 81 Hour: time.Unix(0, 0), 82 }, 83 { 84 CommitPosition: 18, 85 IsSimpleExpectedPass: false, 86 Hour: time.Unix(0, 0), 87 Details: inputbuffer.VerdictDetails{ 88 IsExonerated: true, 89 Runs: []inputbuffer.Run{ 90 { 91 Expected: inputbuffer.ResultCounts{ 92 PassCount: 1, 93 FailCount: 2, 94 CrashCount: 3, 95 AbortCount: 4, 96 }, 97 Unexpected: inputbuffer.ResultCounts{ 98 PassCount: 5, 99 FailCount: 6, 100 CrashCount: 7, 101 AbortCount: 8, 102 }, 103 }, 104 }, 105 }, 106 }, 107 }, 108 }, 109 ColdBufferCapacity: 2000, 110 }, 111 } 112 113 tvb3 := &Entry{ 114 IsNew: true, 115 Project: "proj_3", 116 TestID: "test_id_3", 117 VariantHash: "variant_hash_3", 118 RefHash: []byte("refhash3"), 119 SourceRef: &pb.SourceRef{ 120 System: &pb.SourceRef_Gitiles{ 121 Gitiles: &pb.GitilesRef{ 122 Host: "host_3", 123 Project: "proj_3", 124 Ref: "ref_3", 125 }, 126 }, 127 }, 128 InputBuffer: &inputbuffer.Buffer{ 129 HotBufferCapacity: 100, 130 HotBuffer: inputbuffer.History{ 131 Verdicts: []inputbuffer.PositionVerdict{ 132 { 133 CommitPosition: 20, 134 IsSimpleExpectedPass: true, 135 Hour: time.Unix(0, 0), 136 }, 137 }, 138 }, 139 ColdBufferCapacity: 2000, 140 }, 141 } 142 143 var hs inputbuffer.HistorySerializer 144 mutation1, err := tvb1.ToMutation(&hs) 145 So(err, ShouldBeNil) 146 mutation3, err := tvb3.ToMutation(&hs) 147 So(err, ShouldBeNil) 148 testutil.MustApply(ctx, mutation1, mutation3) 149 150 tvbks := []Key{ 151 makeKey("proj_1", "test_id_1", "variant_hash_1", "refhash1"), 152 makeKey("proj_2", "test_id_2", "variant_hash_2", "refhash2"), 153 makeKey("proj_3", "test_id_3", "variant_hash_3", "refhash3"), 154 } 155 tvbs, err := Read(span.Single(ctx), tvbks) 156 So(err, ShouldBeNil) 157 So(len(tvbs), ShouldEqual, 3) 158 // After inserting, the record should not be new anymore. 159 tvb1.IsNew = false 160 // After decoding, cold buffer should be empty. 161 tvb1.InputBuffer.ColdBuffer = inputbuffer.History{Verdicts: []inputbuffer.PositionVerdict{}} 162 163 So(tvbs[0], ShouldResembleProto, tvb1) 164 165 So(tvbs[1], ShouldBeNil) 166 167 tvb3.IsNew = false 168 tvb3.InputBuffer.ColdBuffer = inputbuffer.History{Verdicts: []inputbuffer.PositionVerdict{}} 169 170 So(tvbs[2], ShouldResembleProto, tvb3) 171 }) 172 173 Convey("Insert and update", t, func() { 174 ctx := testutil.IntegrationTestContext(t) 175 176 // Insert a new record. 177 tvb := &Entry{ 178 IsNew: true, 179 Project: "proj_1", 180 TestID: "test_id_1", 181 VariantHash: "variant_hash_1", 182 RefHash: []byte("githash1"), 183 Variant: &pb.Variant{ 184 Def: map[string]string{ 185 "key1": "val1", 186 "key2": "val2", 187 }, 188 }, 189 SourceRef: &pb.SourceRef{ 190 System: &pb.SourceRef_Gitiles{ 191 Gitiles: &pb.GitilesRef{ 192 Host: "host_1", 193 Project: "proj_1", 194 Ref: "ref_1", 195 }, 196 }, 197 }, 198 InputBuffer: &inputbuffer.Buffer{ 199 HotBufferCapacity: 100, 200 HotBuffer: inputbuffer.History{ 201 Verdicts: []inputbuffer.PositionVerdict{ 202 { 203 CommitPosition: 15, 204 IsSimpleExpectedPass: true, 205 Hour: time.Unix(0, 0), 206 }, 207 }, 208 }, 209 ColdBufferCapacity: 2000, 210 }, 211 } 212 213 var hs inputbuffer.HistorySerializer 214 mutation, err := tvb.ToMutation(&hs) 215 So(err, ShouldBeNil) 216 testutil.MustApply(ctx, mutation) 217 218 // Update the record 219 tvb = &Entry{ 220 Project: "proj_1", 221 TestID: "test_id_1", 222 VariantHash: "variant_hash_1", 223 RefHash: []byte("githash1"), 224 Variant: &pb.Variant{ 225 Def: map[string]string{ 226 "key1": "val1", 227 "key2": "val2", 228 }, 229 }, 230 SourceRef: &pb.SourceRef{ 231 System: &pb.SourceRef_Gitiles{ 232 Gitiles: &pb.GitilesRef{ 233 Host: "host_1", 234 Project: "proj_1", 235 Ref: "ref_1", 236 }, 237 }, 238 }, 239 InputBuffer: &inputbuffer.Buffer{ 240 HotBufferCapacity: 100, 241 HotBuffer: inputbuffer.History{ 242 Verdicts: []inputbuffer.PositionVerdict{ 243 { 244 CommitPosition: 16, 245 IsSimpleExpectedPass: true, 246 Hour: time.Unix(0, 0), 247 }, 248 }, 249 }, 250 ColdBufferCapacity: 2000, 251 ColdBuffer: inputbuffer.History{ 252 Verdicts: []inputbuffer.PositionVerdict{ 253 { 254 CommitPosition: 15, 255 IsSimpleExpectedPass: false, 256 Hour: time.Unix(0, 0), 257 Details: inputbuffer.VerdictDetails{ 258 IsExonerated: false, 259 Runs: []inputbuffer.Run{ 260 { 261 Expected: inputbuffer.ResultCounts{ 262 PassCount: 1, 263 FailCount: 2, 264 CrashCount: 3, 265 AbortCount: 4, 266 }, 267 Unexpected: inputbuffer.ResultCounts{ 268 PassCount: 5, 269 FailCount: 6, 270 CrashCount: 7, 271 AbortCount: 8, 272 }, 273 }, 274 }, 275 }, 276 }, 277 }, 278 }, 279 IsColdBufferDirty: true, 280 }, 281 FinalizingSegment: &cpb.Segment{ 282 State: cpb.SegmentState_FINALIZING, 283 HasStartChangepoint: true, 284 StartPosition: 50, 285 StartHour: timestamppb.New(time.Unix(3600, 0)), 286 StartPositionLowerBound_99Th: 45, 287 StartPositionUpperBound_99Th: 55, 288 FinalizedCounts: &cpb.Counts{ 289 TotalResults: 10, 290 TotalRuns: 10, 291 FlakyRuns: 10, 292 }, 293 }, 294 IsFinalizingSegmentDirty: true, 295 FinalizedSegments: &cpb.Segments{ 296 Segments: []*cpb.Segment{ 297 { 298 State: cpb.SegmentState_FINALIZED, 299 HasStartChangepoint: true, 300 StartPosition: 20, 301 StartHour: timestamppb.New(time.Unix(3600, 0)), 302 StartPositionLowerBound_99Th: 10, 303 StartPositionUpperBound_99Th: 30, 304 EndPosition: 40, 305 EndHour: timestamppb.New(time.Unix(3600, 0)), 306 FinalizedCounts: &cpb.Counts{ 307 TotalResults: 10, 308 TotalRuns: 10, 309 FlakyRuns: 10, 310 UnexpectedResults: 10, 311 ExpectedPassedResults: 1, 312 ExpectedFailedResults: 2, 313 ExpectedCrashedResults: 3, 314 ExpectedAbortedResults: 4, 315 UnexpectedPassedResults: 5, 316 UnexpectedFailedResults: 6, 317 UnexpectedCrashedResults: 7, 318 UnexpectedAbortedResults: 8, 319 }, 320 }, 321 }, 322 }, 323 IsFinalizedSegmentsDirty: true, 324 Statistics: &cpb.Statistics{ 325 HourlyBuckets: []*cpb.Statistics_HourBucket{{ 326 Hour: 100, 327 UnexpectedVerdicts: 1, 328 FlakyVerdicts: 2, 329 TotalVerdicts: 4, 330 }}, 331 }, 332 IsStatisticsDirty: true, 333 } 334 335 mutation, err = tvb.ToMutation(&hs) 336 So(err, ShouldBeNil) 337 testutil.MustApply(ctx, mutation) 338 339 tvbks := []Key{ 340 makeKey("proj_1", "test_id_1", "variant_hash_1", "githash1"), 341 } 342 tvbs, err := Read(span.Single(ctx), tvbks) 343 So(err, ShouldBeNil) 344 So(len(tvbs), ShouldEqual, 1) 345 346 tvb.IsStatisticsDirty = false 347 tvb.IsFinalizedSegmentsDirty = false 348 tvb.IsFinalizingSegmentDirty = false 349 tvb.InputBuffer.IsColdBufferDirty = false 350 351 So(tvbs[0], ShouldResembleProto, tvb) 352 }) 353 } 354 355 func TestInsertToInputBuffer(t *testing.T) { 356 Convey("Insert simple test variant", t, func() { 357 tvb := &Entry{ 358 InputBuffer: &inputbuffer.Buffer{ 359 HotBufferCapacity: 10, 360 ColdBufferCapacity: 100, 361 }, 362 } 363 payload := tu.SamplePayload() 364 sourcesMap := tu.SampleSourcesMap(12) 365 tv := &rdbpb.TestVariant{ 366 Status: rdbpb.TestVariantStatus_EXPECTED, 367 Results: []*rdbpb.TestResultBundle{ 368 { 369 Result: &rdbpb.TestResult{ 370 Expected: true, 371 Status: rdbpb.TestStatus_PASS, 372 }, 373 }, 374 }, 375 SourcesId: "sources_id", 376 } 377 pv, err := ToPositionVerdict(tv, payload, map[string]bool{}, sourcesMap["sources_id"]) 378 So(err, ShouldBeNil) 379 tvb.InsertToInputBuffer(pv) 380 So(len(tvb.InputBuffer.HotBuffer.Verdicts), ShouldEqual, 1) 381 382 So(tvb.InputBuffer.HotBuffer.Verdicts[0], ShouldResemble, inputbuffer.PositionVerdict{ 383 CommitPosition: 12, 384 IsSimpleExpectedPass: true, 385 Hour: payload.PartitionTime.AsTime(), 386 }) 387 }) 388 389 Convey("Insert non-simple test variant", t, func() { 390 tvb := &Entry{ 391 InputBuffer: &inputbuffer.Buffer{ 392 HotBufferCapacity: 10, 393 ColdBufferCapacity: 100, 394 }, 395 } 396 payload := tu.SamplePayload() 397 sourcesMap := tu.SampleSourcesMap(12) 398 tv := &rdbpb.TestVariant{ 399 Status: rdbpb.TestVariantStatus_FLAKY, 400 SourcesId: "sources_id", 401 Results: []*rdbpb.TestResultBundle{ 402 { 403 Result: &rdbpb.TestResult{ 404 Name: "invocations/run-1/tests/abc", 405 Expected: true, 406 Status: rdbpb.TestStatus_PASS, 407 }, 408 }, 409 { 410 Result: &rdbpb.TestResult{ 411 Name: "invocations/run-1/tests/abc", 412 Expected: true, 413 Status: rdbpb.TestStatus_FAIL, 414 }, 415 }, 416 { 417 Result: &rdbpb.TestResult{ 418 Name: "invocations/run-1/tests/abc", 419 Expected: true, 420 Status: rdbpb.TestStatus_CRASH, 421 }, 422 }, 423 { 424 Result: &rdbpb.TestResult{ 425 Name: "invocations/run-2/tests/abc", 426 Expected: true, 427 Status: rdbpb.TestStatus_ABORT, 428 }, 429 }, 430 { 431 Result: &rdbpb.TestResult{ 432 Name: "invocations/run-3/tests/abc", 433 Expected: false, 434 Status: rdbpb.TestStatus_PASS, 435 }, 436 }, 437 { 438 Result: &rdbpb.TestResult{ 439 Name: "invocations/run-3/tests/abc", 440 Expected: false, 441 Status: rdbpb.TestStatus_FAIL, 442 }, 443 }, 444 { 445 Result: &rdbpb.TestResult{ 446 Name: "invocations/run-4/tests/abc", 447 Expected: false, 448 Status: rdbpb.TestStatus_CRASH, 449 }, 450 }, 451 { 452 Result: &rdbpb.TestResult{ 453 Name: "invocations/run-4/tests/abc", 454 Expected: false, 455 Status: rdbpb.TestStatus_ABORT, 456 }, 457 }, 458 }, 459 } 460 duplicateMap := map[string]bool{ 461 "run-1": true, 462 "run-3": true, 463 } 464 pv, err := ToPositionVerdict(tv, payload, duplicateMap, sourcesMap["sources_id"]) 465 So(err, ShouldBeNil) 466 tvb.InsertToInputBuffer(pv) 467 So(len(tvb.InputBuffer.HotBuffer.Verdicts), ShouldEqual, 1) 468 469 So(tvb.InputBuffer.HotBuffer.Verdicts[0], ShouldResemble, inputbuffer.PositionVerdict{ 470 CommitPosition: 12, 471 IsSimpleExpectedPass: false, 472 Hour: payload.PartitionTime.AsTime(), 473 Details: inputbuffer.VerdictDetails{ 474 IsExonerated: false, 475 Runs: []inputbuffer.Run{ 476 { 477 Unexpected: inputbuffer.ResultCounts{ 478 CrashCount: 1, 479 AbortCount: 1, 480 }, 481 IsDuplicate: false, 482 }, 483 { 484 Expected: inputbuffer.ResultCounts{ 485 AbortCount: 1, 486 }, 487 IsDuplicate: false, 488 }, 489 { 490 Unexpected: inputbuffer.ResultCounts{ 491 PassCount: 1, 492 FailCount: 1, 493 }, 494 IsDuplicate: true, 495 }, 496 { 497 Expected: inputbuffer.ResultCounts{ 498 PassCount: 1, 499 FailCount: 1, 500 CrashCount: 1, 501 }, 502 IsDuplicate: true, 503 }, 504 }, 505 }, 506 }) 507 }) 508 } 509 510 func TestUpdateOutputBuffer(t *testing.T) { 511 Convey("No existing finalizing segment", t, func() { 512 tvb := Entry{} 513 evictedSegments := []inputbuffer.EvictedSegment{ 514 { 515 Segment: &cpb.Segment{ 516 State: cpb.SegmentState_FINALIZED, 517 StartPosition: 1, 518 EndPosition: 10, 519 FinalizedCounts: &cpb.Counts{ 520 TotalResults: 10, 521 TotalRuns: 10, 522 TotalVerdicts: 10, 523 ExpectedPassedResults: 1, 524 ExpectedFailedResults: 2, 525 ExpectedCrashedResults: 3, 526 ExpectedAbortedResults: 4, 527 UnexpectedPassedResults: 5, 528 UnexpectedFailedResults: 6, 529 UnexpectedCrashedResults: 7, 530 UnexpectedAbortedResults: 8, 531 }, 532 }, 533 Verdicts: []inputbuffer.PositionVerdict{ 534 { 535 CommitPosition: 2, 536 Hour: time.Unix(10*3600, 0), 537 IsSimpleExpectedPass: true, 538 }, 539 }, 540 }, 541 { 542 Segment: &cpb.Segment{ 543 State: cpb.SegmentState_FINALIZING, 544 StartPosition: 11, 545 EndPosition: 30, 546 FinalizedCounts: &cpb.Counts{ 547 TotalResults: 20, 548 TotalRuns: 20, 549 TotalVerdicts: 20, 550 }, 551 }, 552 Verdicts: []inputbuffer.PositionVerdict{ 553 { 554 CommitPosition: 11, 555 Hour: time.Unix(11*3600, 0), 556 IsSimpleExpectedPass: true, 557 }, 558 { 559 CommitPosition: 12, 560 Hour: time.Unix((100000-StatisticsRetentionDays*24)*3600, 0), 561 IsSimpleExpectedPass: true, 562 }, 563 { 564 CommitPosition: 13, 565 Hour: time.Unix((100000-StatisticsRetentionDays*24+1)*3600, 0), 566 IsSimpleExpectedPass: false, 567 Details: inputbuffer.VerdictDetails{ 568 Runs: []inputbuffer.Run{ 569 { 570 Expected: inputbuffer.ResultCounts{ 571 PassCount: 1, 572 }, 573 Unexpected: inputbuffer.ResultCounts{ 574 FailCount: 1, 575 }, 576 }, 577 }, 578 }, 579 }, 580 { 581 CommitPosition: 11, 582 Hour: time.Unix(100000*3600, 0), 583 IsSimpleExpectedPass: false, 584 Details: inputbuffer.VerdictDetails{ 585 Runs: []inputbuffer.Run{ 586 { 587 Unexpected: inputbuffer.ResultCounts{ 588 CrashCount: 1, 589 }, 590 }, 591 }, 592 }, 593 }, 594 }, 595 }, 596 } 597 tvb.UpdateOutputBuffer(evictedSegments) 598 599 So(tvb.FinalizingSegment, ShouldNotBeNil) 600 So(tvb.FinalizingSegment, ShouldResembleProto, evictedSegments[1].Segment) 601 So(tvb.IsFinalizingSegmentDirty, ShouldBeTrue) 602 603 So(len(tvb.FinalizedSegments.Segments), ShouldEqual, 1) 604 So(tvb.FinalizedSegments.Segments[0], ShouldResembleProto, evictedSegments[0].Segment) 605 So(tvb.IsFinalizedSegmentsDirty, ShouldBeTrue) 606 607 So(tvb.Statistics, ShouldResembleProto, &cpb.Statistics{ 608 HourlyBuckets: []*cpb.Statistics_HourBucket{ 609 // Confirm that buckets for hours 11 and (100000 - StatisticsRetentionDays*24) 610 // are not present due to retention policies. 611 { 612 Hour: (100000 - StatisticsRetentionDays*24 + 1), 613 TotalVerdicts: 1, 614 FlakyVerdicts: 1, 615 }, 616 { 617 Hour: 100000, 618 TotalVerdicts: 1, 619 UnexpectedVerdicts: 1, 620 }, 621 }, 622 }) 623 So(tvb.IsStatisticsDirty, ShouldBeTrue) 624 }) 625 626 Convey("Combine finalizing segment with finalizing segment", t, func() { 627 tvb := Entry{ 628 FinalizingSegment: &cpb.Segment{ 629 State: cpb.SegmentState_FINALIZING, 630 StartPosition: 100, 631 StartHour: timestamppb.New(time.Unix(3600, 0)), 632 HasStartChangepoint: true, 633 StartPositionLowerBound_99Th: 90, 634 StartPositionUpperBound_99Th: 110, 635 FinalizedCounts: &cpb.Counts{ 636 TotalResults: 30, 637 UnexpectedResults: 5, 638 TotalRuns: 20, 639 UnexpectedUnretriedRuns: 2, 640 UnexpectedAfterRetryRuns: 3, 641 FlakyRuns: 4, 642 TotalVerdicts: 10, 643 UnexpectedVerdicts: 1, 644 FlakyVerdicts: 2, 645 ExpectedPassedResults: 1, 646 ExpectedFailedResults: 2, 647 ExpectedCrashedResults: 3, 648 ExpectedAbortedResults: 4, 649 UnexpectedPassedResults: 5, 650 UnexpectedFailedResults: 6, 651 UnexpectedCrashedResults: 7, 652 UnexpectedAbortedResults: 8, 653 }, 654 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(7*3600, 0)), 655 }, 656 } 657 evictedSegments := []inputbuffer.EvictedSegment{ 658 { 659 Segment: &cpb.Segment{ 660 661 State: cpb.SegmentState_FINALIZING, 662 StartPosition: 200, 663 StartHour: timestamppb.New(time.Unix(100*3600, 0)), 664 HasStartChangepoint: false, 665 StartPositionLowerBound_99Th: 190, 666 StartPositionUpperBound_99Th: 210, 667 FinalizedCounts: &cpb.Counts{ 668 TotalResults: 50, 669 UnexpectedResults: 3, 670 TotalRuns: 40, 671 UnexpectedUnretriedRuns: 5, 672 UnexpectedAfterRetryRuns: 6, 673 FlakyRuns: 7, 674 TotalVerdicts: 20, 675 UnexpectedVerdicts: 3, 676 FlakyVerdicts: 2, 677 ExpectedPassedResults: 1, 678 ExpectedFailedResults: 2, 679 ExpectedCrashedResults: 3, 680 ExpectedAbortedResults: 4, 681 UnexpectedPassedResults: 5, 682 UnexpectedFailedResults: 6, 683 UnexpectedCrashedResults: 7, 684 UnexpectedAbortedResults: 8, 685 }, 686 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 687 }, 688 // Verdicts are not relevant to this test. 689 Verdicts: []inputbuffer.PositionVerdict{}, 690 }, 691 } 692 tvb.UpdateOutputBuffer(evictedSegments) 693 So(tvb.FinalizedSegments, ShouldBeNil) 694 So(tvb.FinalizingSegment, ShouldNotBeNil) 695 696 expected := &cpb.Segment{ 697 State: cpb.SegmentState_FINALIZING, 698 StartPosition: 100, 699 StartHour: timestamppb.New(time.Unix(3600, 0)), 700 HasStartChangepoint: true, 701 StartPositionLowerBound_99Th: 90, 702 StartPositionUpperBound_99Th: 110, 703 FinalizedCounts: &cpb.Counts{ 704 TotalResults: 80, 705 UnexpectedResults: 8, 706 TotalRuns: 60, 707 UnexpectedUnretriedRuns: 7, 708 UnexpectedAfterRetryRuns: 9, 709 FlakyRuns: 11, 710 TotalVerdicts: 30, 711 UnexpectedVerdicts: 4, 712 FlakyVerdicts: 4, 713 ExpectedPassedResults: 2, 714 ExpectedFailedResults: 4, 715 ExpectedCrashedResults: 6, 716 ExpectedAbortedResults: 8, 717 UnexpectedPassedResults: 10, 718 UnexpectedFailedResults: 12, 719 UnexpectedCrashedResults: 14, 720 UnexpectedAbortedResults: 16, 721 }, 722 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 723 } 724 So(tvb.FinalizingSegment, ShouldResembleProto, expected) 725 }) 726 727 Convey("Combine finalizing segment with finalized segment", t, func() { 728 tvb := Entry{ 729 FinalizingSegment: &cpb.Segment{ 730 State: cpb.SegmentState_FINALIZING, 731 StartPosition: 100, 732 StartHour: timestamppb.New(time.Unix(3600, 0)), 733 HasStartChangepoint: true, 734 StartPositionLowerBound_99Th: 90, 735 StartPositionUpperBound_99Th: 110, 736 FinalizedCounts: &cpb.Counts{ 737 TotalResults: 30, 738 UnexpectedResults: 5, 739 TotalRuns: 20, 740 UnexpectedUnretriedRuns: 2, 741 UnexpectedAfterRetryRuns: 3, 742 FlakyRuns: 4, 743 TotalVerdicts: 10, 744 UnexpectedVerdicts: 1, 745 FlakyVerdicts: 2, 746 ExpectedPassedResults: 1, 747 ExpectedFailedResults: 2, 748 ExpectedCrashedResults: 3, 749 ExpectedAbortedResults: 4, 750 UnexpectedPassedResults: 5, 751 UnexpectedFailedResults: 6, 752 UnexpectedCrashedResults: 7, 753 UnexpectedAbortedResults: 8, 754 }, 755 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(7*3600, 0)), 756 }, 757 } 758 evictedSegments := []inputbuffer.EvictedSegment{ 759 { 760 Segment: &cpb.Segment{ 761 State: cpb.SegmentState_FINALIZED, 762 StartPosition: 200, 763 StartHour: timestamppb.New(time.Unix(100*3600, 0)), 764 HasStartChangepoint: false, 765 StartPositionLowerBound_99Th: 190, 766 StartPositionUpperBound_99Th: 210, 767 EndPosition: 400, 768 EndHour: timestamppb.New(time.Unix(400*3600, 0)), 769 FinalizedCounts: &cpb.Counts{ 770 TotalResults: 50, 771 UnexpectedResults: 3, 772 TotalRuns: 40, 773 UnexpectedUnretriedRuns: 5, 774 UnexpectedAfterRetryRuns: 6, 775 FlakyRuns: 7, 776 TotalVerdicts: 20, 777 UnexpectedVerdicts: 3, 778 FlakyVerdicts: 2, 779 ExpectedPassedResults: 1, 780 ExpectedFailedResults: 2, 781 ExpectedCrashedResults: 3, 782 ExpectedAbortedResults: 4, 783 UnexpectedPassedResults: 5, 784 UnexpectedFailedResults: 6, 785 UnexpectedCrashedResults: 7, 786 UnexpectedAbortedResults: 8, 787 }, 788 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 789 }, 790 // Verdicts are not relevant to this test. 791 Verdicts: []inputbuffer.PositionVerdict{}, 792 }, 793 { 794 Segment: &cpb.Segment{ 795 State: cpb.SegmentState_FINALIZING, 796 StartPosition: 500, 797 EndPosition: 800, 798 FinalizedCounts: &cpb.Counts{ 799 TotalResults: 20, 800 TotalRuns: 20, 801 TotalVerdicts: 20, 802 }, 803 }, 804 // Verdicts are not relevant to this test. 805 Verdicts: []inputbuffer.PositionVerdict{}, 806 }, 807 } 808 tvb.UpdateOutputBuffer(evictedSegments) 809 So(len(tvb.FinalizedSegments.Segments), ShouldEqual, 1) 810 So(tvb.FinalizingSegment, ShouldNotBeNil) 811 So(tvb.FinalizingSegment, ShouldResembleProto, evictedSegments[1].Segment) 812 expected := &cpb.Segment{ 813 State: cpb.SegmentState_FINALIZED, 814 StartPosition: 100, 815 StartHour: timestamppb.New(time.Unix(3600, 0)), 816 HasStartChangepoint: true, 817 StartPositionLowerBound_99Th: 90, 818 StartPositionUpperBound_99Th: 110, 819 EndPosition: 400, 820 EndHour: timestamppb.New(time.Unix(400*3600, 0)), 821 FinalizedCounts: &cpb.Counts{ 822 TotalResults: 80, 823 UnexpectedResults: 8, 824 TotalRuns: 60, 825 UnexpectedUnretriedRuns: 7, 826 UnexpectedAfterRetryRuns: 9, 827 FlakyRuns: 11, 828 TotalVerdicts: 30, 829 UnexpectedVerdicts: 4, 830 FlakyVerdicts: 4, 831 ExpectedPassedResults: 2, 832 ExpectedFailedResults: 4, 833 ExpectedCrashedResults: 6, 834 ExpectedAbortedResults: 8, 835 UnexpectedPassedResults: 10, 836 UnexpectedFailedResults: 12, 837 UnexpectedCrashedResults: 14, 838 UnexpectedAbortedResults: 16, 839 }, 840 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 841 } 842 So(tvb.FinalizedSegments.Segments[0], ShouldResembleProto, expected) 843 }) 844 845 Convey("Combine finalizing segment with finalized segment, with a token of finalizing segment in input buffer", t, func() { 846 tvb := Entry{ 847 FinalizingSegment: &cpb.Segment{ 848 State: cpb.SegmentState_FINALIZING, 849 StartPosition: 100, 850 StartHour: timestamppb.New(time.Unix(3600, 0)), 851 HasStartChangepoint: true, 852 StartPositionLowerBound_99Th: 90, 853 StartPositionUpperBound_99Th: 110, 854 FinalizedCounts: &cpb.Counts{ 855 TotalResults: 30, 856 UnexpectedResults: 5, 857 TotalRuns: 20, 858 UnexpectedUnretriedRuns: 2, 859 UnexpectedAfterRetryRuns: 3, 860 FlakyRuns: 4, 861 TotalVerdicts: 10, 862 UnexpectedVerdicts: 1, 863 FlakyVerdicts: 2, 864 ExpectedPassedResults: 1, 865 ExpectedFailedResults: 2, 866 ExpectedCrashedResults: 3, 867 ExpectedAbortedResults: 4, 868 UnexpectedPassedResults: 5, 869 UnexpectedFailedResults: 6, 870 UnexpectedCrashedResults: 7, 871 UnexpectedAbortedResults: 8, 872 }, 873 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(7*3600, 0)), 874 }, 875 } 876 evictedSegments := []inputbuffer.EvictedSegment{ 877 { 878 Segment: &cpb.Segment{ 879 State: cpb.SegmentState_FINALIZED, 880 StartPosition: 200, 881 StartHour: timestamppb.New(time.Unix(100*3600, 0)), 882 HasStartChangepoint: false, 883 StartPositionLowerBound_99Th: 190, 884 StartPositionUpperBound_99Th: 210, 885 EndPosition: 400, 886 EndHour: timestamppb.New(time.Unix(400*3600, 0)), 887 FinalizedCounts: &cpb.Counts{ 888 TotalResults: 50, 889 UnexpectedResults: 3, 890 TotalRuns: 40, 891 UnexpectedUnretriedRuns: 5, 892 UnexpectedAfterRetryRuns: 6, 893 FlakyRuns: 7, 894 TotalVerdicts: 20, 895 UnexpectedVerdicts: 3, 896 FlakyVerdicts: 2, 897 }, 898 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 899 }, 900 // Verdicts are not relevant to this test. 901 Verdicts: []inputbuffer.PositionVerdict{}, 902 }, 903 { 904 Segment: &cpb.Segment{ 905 State: cpb.SegmentState_FINALIZING, 906 StartPosition: 500, 907 StartHour: timestamppb.New(time.Unix(500*3600, 0)), 908 HasStartChangepoint: true, 909 StartPositionLowerBound_99Th: 490, 910 StartPositionUpperBound_99Th: 510, 911 FinalizedCounts: &cpb.Counts{}, 912 }, 913 // Verdicts are not relevant to this test. 914 Verdicts: []inputbuffer.PositionVerdict{}, 915 }, 916 } 917 tvb.UpdateOutputBuffer(evictedSegments) 918 So(len(tvb.FinalizedSegments.Segments), ShouldEqual, 1) 919 So(tvb.FinalizingSegment, ShouldNotBeNil) 920 expected := &cpb.Segment{ 921 State: cpb.SegmentState_FINALIZED, 922 StartPosition: 100, 923 StartHour: timestamppb.New(time.Unix(3600, 0)), 924 HasStartChangepoint: true, 925 StartPositionLowerBound_99Th: 90, 926 StartPositionUpperBound_99Th: 110, 927 EndPosition: 400, 928 EndHour: timestamppb.New(time.Unix(400*3600, 0)), 929 FinalizedCounts: &cpb.Counts{ 930 TotalResults: 80, 931 UnexpectedResults: 8, 932 TotalRuns: 60, 933 UnexpectedUnretriedRuns: 7, 934 UnexpectedAfterRetryRuns: 9, 935 FlakyRuns: 11, 936 TotalVerdicts: 30, 937 UnexpectedVerdicts: 4, 938 FlakyVerdicts: 4, 939 ExpectedPassedResults: 1, 940 ExpectedFailedResults: 2, 941 ExpectedCrashedResults: 3, 942 ExpectedAbortedResults: 4, 943 UnexpectedPassedResults: 5, 944 UnexpectedFailedResults: 6, 945 UnexpectedCrashedResults: 7, 946 UnexpectedAbortedResults: 8, 947 }, 948 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 949 } 950 So(tvb.FinalizedSegments.Segments[0], ShouldResembleProto, expected) 951 So(tvb.FinalizingSegment, ShouldResembleProto, evictedSegments[1].Segment) 952 }) 953 954 Convey("Should panic if no finalizing segment in evicted segments", t, func() { 955 tvb := Entry{ 956 FinalizingSegment: &cpb.Segment{ 957 State: cpb.SegmentState_FINALIZING, 958 StartPosition: 100, 959 StartHour: timestamppb.New(time.Unix(3600, 0)), 960 HasStartChangepoint: true, 961 StartPositionLowerBound_99Th: 90, 962 StartPositionUpperBound_99Th: 110, 963 FinalizedCounts: &cpb.Counts{ 964 TotalResults: 30, 965 UnexpectedResults: 5, 966 TotalRuns: 20, 967 UnexpectedUnretriedRuns: 2, 968 UnexpectedAfterRetryRuns: 3, 969 FlakyRuns: 4, 970 TotalVerdicts: 10, 971 UnexpectedVerdicts: 1, 972 FlakyVerdicts: 2, 973 }, 974 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(7*3600, 0)), 975 }, 976 } 977 evictedSegments := []inputbuffer.EvictedSegment{ 978 { 979 Segment: &cpb.Segment{ 980 State: cpb.SegmentState_FINALIZED, 981 StartPosition: 200, 982 StartHour: timestamppb.New(time.Unix(100*3600, 0)), 983 HasStartChangepoint: false, 984 StartPositionLowerBound_99Th: 190, 985 StartPositionUpperBound_99Th: 210, 986 EndPosition: 400, 987 EndHour: timestamppb.New(time.Unix(400*3600, 0)), 988 FinalizedCounts: &cpb.Counts{ 989 TotalResults: 50, 990 UnexpectedResults: 3, 991 TotalRuns: 40, 992 UnexpectedUnretriedRuns: 5, 993 UnexpectedAfterRetryRuns: 6, 994 FlakyRuns: 7, 995 TotalVerdicts: 20, 996 UnexpectedVerdicts: 3, 997 FlakyVerdicts: 2, 998 }, 999 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 1000 }, 1001 // Verdicts are not relevant to this test. 1002 Verdicts: []inputbuffer.PositionVerdict{}, 1003 }, 1004 } 1005 f := func() { tvb.UpdateOutputBuffer(evictedSegments) } 1006 So(f, ShouldPanic) 1007 }) 1008 1009 Convey("Statistics should be updated following eviction", t, func() { 1010 tvb := Entry{ 1011 Statistics: &cpb.Statistics{ 1012 HourlyBuckets: []*cpb.Statistics_HourBucket{ 1013 { 1014 Hour: 999, 1015 UnexpectedVerdicts: 1, 1016 FlakyVerdicts: 2, 1017 TotalVerdicts: 3, 1018 }, 1019 { 1020 Hour: 1000, 1021 FlakyVerdicts: 10, 1022 TotalVerdicts: 10, 1023 }, 1024 }, 1025 }, 1026 } 1027 evictedSegments := []inputbuffer.EvictedSegment{ 1028 { 1029 Segment: &cpb.Segment{ 1030 1031 State: cpb.SegmentState_FINALIZING, 1032 StartPosition: 200, 1033 StartHour: timestamppb.New(time.Unix(100*3600, 0)), 1034 HasStartChangepoint: false, 1035 StartPositionLowerBound_99Th: 190, 1036 StartPositionUpperBound_99Th: 210, 1037 FinalizedCounts: &cpb.Counts{}, 1038 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(10*3600, 0)), 1039 }, 1040 Verdicts: []inputbuffer.PositionVerdict{ 1041 // Expected verdict. 1042 { 1043 CommitPosition: 190, 1044 Hour: time.Unix(1000*3600, 0), 1045 IsSimpleExpectedPass: true, 1046 }, 1047 // Expected verdict. 1048 { 1049 CommitPosition: 191, 1050 Hour: time.Unix(1000*3600, 0), 1051 Details: inputbuffer.VerdictDetails{ 1052 Runs: []inputbuffer.Run{ 1053 { 1054 Expected: inputbuffer.ResultCounts{ 1055 PassCount: 2, 1056 }, 1057 }, 1058 }, 1059 }, 1060 }, 1061 // Flaky verdict. 1062 { 1063 CommitPosition: 192, 1064 Hour: time.Unix(992*3600, 0), 1065 Details: inputbuffer.VerdictDetails{ 1066 Runs: []inputbuffer.Run{ 1067 { 1068 Expected: inputbuffer.ResultCounts{ 1069 PassCount: 1, 1070 }, 1071 }, 1072 { 1073 IsDuplicate: true, 1074 Unexpected: inputbuffer.ResultCounts{ 1075 FailCount: 1, 1076 }, 1077 }, 1078 }, 1079 }, 1080 }, 1081 // Unexpected verdict. 1082 { 1083 CommitPosition: 193, 1084 Hour: time.Unix(991*3600, 0), 1085 Details: inputbuffer.VerdictDetails{ 1086 Runs: []inputbuffer.Run{ 1087 { 1088 Unexpected: inputbuffer.ResultCounts{ 1089 FailCount: 1, 1090 }, 1091 }, 1092 { 1093 Unexpected: inputbuffer.ResultCounts{ 1094 FailCount: 1, 1095 }, 1096 }, 1097 }, 1098 }, 1099 }, 1100 // Verdict which is (just) within the retention policy. 1101 { 1102 CommitPosition: 194, 1103 Hour: time.Unix((1000-StatisticsRetentionDays*24+1)*3600, 0), 1104 Details: inputbuffer.VerdictDetails{ 1105 Runs: []inputbuffer.Run{ 1106 { 1107 Unexpected: inputbuffer.ResultCounts{ 1108 FailCount: 1, 1109 }, 1110 }, 1111 { 1112 Unexpected: inputbuffer.ResultCounts{ 1113 FailCount: 1, 1114 }, 1115 }, 1116 }, 1117 }, 1118 }, 1119 // Verdict which is from too old a bucket. 1120 { 1121 CommitPosition: 194, 1122 Hour: time.Unix((1000-StatisticsRetentionDays*24)*3600, 0), 1123 Details: inputbuffer.VerdictDetails{ 1124 Runs: []inputbuffer.Run{ 1125 { 1126 Unexpected: inputbuffer.ResultCounts{ 1127 FailCount: 1, 1128 }, 1129 }, 1130 }, 1131 }, 1132 }, 1133 }, 1134 }, 1135 } 1136 tvb.UpdateOutputBuffer(evictedSegments) 1137 1138 expected := &cpb.Statistics{ 1139 HourlyBuckets: []*cpb.Statistics_HourBucket{ 1140 { 1141 Hour: (1000 - StatisticsRetentionDays*24 + 1), 1142 UnexpectedVerdicts: 1, 1143 TotalVerdicts: 1, 1144 }, 1145 { 1146 Hour: 991, 1147 UnexpectedVerdicts: 1, 1148 TotalVerdicts: 1, 1149 }, 1150 { 1151 Hour: 992, 1152 FlakyVerdicts: 1, 1153 TotalVerdicts: 1, 1154 }, 1155 { 1156 Hour: 999, 1157 UnexpectedVerdicts: 1, 1158 FlakyVerdicts: 2, 1159 TotalVerdicts: 3, 1160 }, 1161 { 1162 Hour: 1000, 1163 FlakyVerdicts: 10, 1164 TotalVerdicts: 12, 1165 }, 1166 }, 1167 } 1168 So(tvb.Statistics, ShouldResembleProto, expected) 1169 So(tvb.IsStatisticsDirty, ShouldBeTrue) 1170 }) 1171 1172 Convey("Output buffer should not be updated if there is no eviction", t, func() { 1173 tvb := Entry{} 1174 tvb.UpdateOutputBuffer([]inputbuffer.EvictedSegment{}) 1175 So(tvb.IsStatisticsDirty, ShouldBeFalse) 1176 So(tvb.IsFinalizingSegmentDirty, ShouldBeFalse) 1177 So(tvb.IsFinalizedSegmentsDirty, ShouldBeFalse) 1178 }) 1179 } 1180 1181 func makeKey(proj string, testID string, variantHash string, refHash RefHash) Key { 1182 return Key{ 1183 Project: proj, 1184 TestID: testID, 1185 VariantHash: variantHash, 1186 RefHash: refHash, 1187 } 1188 } 1189 1190 func BenchmarkEncodeSegments(b *testing.B) { 1191 // Result when test added: 1192 // cpu: Intel(R) Xeon(R) CPU @ 2.00GHz 1193 // BenchmarkEncodeSegments-96 3343 328663 ns/op 14598 B/op 3 allocs/op 1194 b.StopTimer() 1195 segs := testSegments() 1196 b.StartTimer() 1197 for i := 0; i < b.N; i++ { 1198 _, err := EncodeSegments(segs) 1199 if err != nil { 1200 panic(err) 1201 } 1202 } 1203 } 1204 1205 func BenchmarkDecodeSegments(b *testing.B) { 1206 // Result when test added: 1207 // cpu: Intel(R) Xeon(R) CPU @ 2.00GHz 1208 // BenchmarkDecodeSegments-96 10663 111653 ns/op 53190 B/op 510 allocs/op 1209 b.StopTimer() 1210 segs := testSegments() 1211 encodedSegs, err := EncodeSegments(segs) 1212 if err != nil { 1213 panic(err) 1214 } 1215 b.StartTimer() 1216 for i := 0; i < b.N; i++ { 1217 _, err := DecodeSegments(encodedSegs) 1218 if err != nil { 1219 panic(err) 1220 } 1221 } 1222 } 1223 1224 func BenchmarkEncodeStatistics(b *testing.B) { 1225 // Result when test added: 1226 // cpu: Intel(R) Xeon(R) CPU @ 2.00GHz 1227 // BenchmarkEncodeStatistics-96 15578 71829 ns/op 8321 B/op 3 allocs/op 1228 b.StopTimer() 1229 stats := testStatistics() 1230 b.StartTimer() 1231 for i := 0; i < b.N; i++ { 1232 _, err := EncodeStatistics(stats) 1233 if err != nil { 1234 panic(err) 1235 } 1236 } 1237 } 1238 1239 func BenchmarkDecodeStatistics(b *testing.B) { 1240 // Result when test added: 1241 // cpu: Intel(R) Xeon(R) CPU @ 2.00GHz 1242 // BenchmarkDecodeStatistics-96 18616 61122 ns/op 34236 B/op 276 allocs/op 1243 b.StopTimer() 1244 stats := testStatistics() 1245 encodedStats, err := EncodeStatistics(stats) 1246 if err != nil { 1247 panic(err) 1248 } 1249 b.StartTimer() 1250 for i := 0; i < b.N; i++ { 1251 _, err := DecodeStatistics(encodedStats) 1252 if err != nil { 1253 panic(err) 1254 } 1255 } 1256 } 1257 1258 func testStatistics() *cpb.Statistics { 1259 var buckets []*cpb.Statistics_HourBucket 1260 for i := 0; i < StatisticsRetentionDays*24; i++ { 1261 buckets = append(buckets, &cpb.Statistics_HourBucket{ 1262 Hour: int64(i), 1263 UnexpectedVerdicts: int64(i + 1), 1264 FlakyVerdicts: int64(i + 2), 1265 TotalVerdicts: int64(2*i + 3), 1266 }) 1267 } 1268 return &cpb.Statistics{HourlyBuckets: buckets} 1269 } 1270 1271 func testSegments() *cpb.Segments { 1272 var segments []*cpb.Segment 1273 for i := 0; i < 100; i++ { 1274 segments = append(segments, &cpb.Segment{ 1275 State: cpb.SegmentState_FINALIZED, 1276 HasStartChangepoint: true, 1277 StartPosition: int64(162*i + 5), 1278 StartPositionLowerBound_99Th: int64(162 * i), 1279 StartPositionUpperBound_99Th: int64(162*i + 10), 1280 EndPosition: int64(162 * (i + 1)), 1281 StartHour: timestamppb.New(time.Unix(int64(i*12*3600), 0)), 1282 EndHour: timestamppb.New(time.Unix(int64((i+1)*12*3600), 0)), 1283 MostRecentUnexpectedResultHour: timestamppb.New(time.Unix(int64((i+1)*9*3600), 0)), 1284 FinalizedCounts: &cpb.Counts{ 1285 UnexpectedResults: int64(i), 1286 TotalResults: int64(i + 1), 1287 1288 UnexpectedUnretriedRuns: int64(i + 2), 1289 UnexpectedAfterRetryRuns: int64(i + 3), 1290 FlakyRuns: int64(i + 4), 1291 TotalRuns: int64(4*i + 5), 1292 1293 UnexpectedVerdicts: int64(i + 6), 1294 FlakyVerdicts: int64(i + 7), 1295 TotalVerdicts: int64(2*i + 8), 1296 }, 1297 }) 1298 } 1299 return &cpb.Segments{ 1300 Segments: segments, 1301 } 1302 }