go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/frontend/ui/src/tools/failure_tools.test.ts (about)

     1  // Copyright 2022 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  /* eslint-disable jest/no-conditional-expect */
    15  
    16  
    17  import { DistinctClusterFailure } from '@/proto/go.chromium.org/luci/analysis/proto/v1/clusters.pb';
    18  import {
    19    impactFilterNamed,
    20    newMockFailure,
    21    newMockGroup,
    22  } from '@/testing_tools/mocks/failures_mock';
    23  import {
    24    FailureGroup,
    25    groupFailures,
    26    rejectedIngestedInvocationIdsExtractor,
    27    rejectedPresubmitRunIdsExtractor,
    28    sortFailureGroups,
    29    treeDistinctValues,
    30  } from './failures_tools';
    31  
    32  interface ExtractorTestCase {
    33      failure: DistinctClusterFailure;
    34      filter: string;
    35      shouldExtractIngestedInvocationId: boolean;
    36      shouldExtractPresubmitRunId: boolean;
    37  }
    38  
    39  describe.each<ExtractorTestCase>([{
    40    failure: newMockFailure().build(),
    41    filter: 'None (Actual Impact)',
    42    shouldExtractIngestedInvocationId: false,
    43    shouldExtractPresubmitRunId: false,
    44  }, {
    45    failure: newMockFailure().exonerateNotCritical().build(),
    46    filter: 'None (Actual Impact)',
    47    shouldExtractIngestedInvocationId: false,
    48    shouldExtractPresubmitRunId: false,
    49  }, {
    50    failure: newMockFailure().exonerateOccursOnOtherCLs().build(),
    51    filter: 'None (Actual Impact)',
    52    shouldExtractIngestedInvocationId: false,
    53    shouldExtractPresubmitRunId: false,
    54  }, {
    55    failure: newMockFailure().exonerateNotCritical().build(),
    56    filter: 'None (Actual Impact)',
    57    shouldExtractIngestedInvocationId: false,
    58    shouldExtractPresubmitRunId: false,
    59  }, {
    60    failure: newMockFailure().ingestedInvocationBlocked().build(),
    61    filter: 'None (Actual Impact)',
    62    shouldExtractIngestedInvocationId: false,
    63    shouldExtractPresubmitRunId: false,
    64  }, {
    65    failure: newMockFailure().ingestedInvocationBlocked().buildFailed().build(),
    66    filter: 'None (Actual Impact)',
    67    shouldExtractIngestedInvocationId: true,
    68    shouldExtractPresubmitRunId: true,
    69  }, {
    70    failure: newMockFailure().ingestedInvocationBlocked().buildFailed().notPresubmitCritical().build(),
    71    filter: 'None (Actual Impact)',
    72    shouldExtractIngestedInvocationId: true,
    73    shouldExtractPresubmitRunId: false,
    74  }, {
    75    failure: newMockFailure().ingestedInvocationBlocked().buildFailed().dryRun().build(),
    76    filter: 'None (Actual Impact)',
    77    shouldExtractIngestedInvocationId: true,
    78    shouldExtractPresubmitRunId: false,
    79  }, {
    80    failure: newMockFailure().ingestedInvocationBlocked().buildFailed().exonerateOccursOnOtherCLs().build(),
    81    filter: 'None (Actual Impact)',
    82    shouldExtractIngestedInvocationId: false,
    83    shouldExtractPresubmitRunId: false,
    84  }, {
    85    failure: newMockFailure().ingestedInvocationBlocked().buildFailed().exonerateNotCritical().build(),
    86    filter: 'None (Actual Impact)',
    87    shouldExtractIngestedInvocationId: false,
    88    shouldExtractPresubmitRunId: false,
    89  }, {
    90    failure: newMockFailure().exonerateOccursOnOtherCLs().build(),
    91    filter: 'None (Actual Impact)',
    92    shouldExtractIngestedInvocationId: false,
    93    shouldExtractPresubmitRunId: false,
    94  }, {
    95    failure: newMockFailure().build(),
    96    filter: 'Without LUCI Analysis Exoneration',
    97    shouldExtractIngestedInvocationId: false,
    98    shouldExtractPresubmitRunId: false,
    99  }, {
   100    failure: newMockFailure().ingestedInvocationBlocked().build(),
   101    filter: 'Without LUCI Analysis Exoneration',
   102    shouldExtractIngestedInvocationId: true,
   103    shouldExtractPresubmitRunId: true,
   104  }, {
   105    failure: newMockFailure().ingestedInvocationBlocked().notPresubmitCritical().build(),
   106    filter: 'Without LUCI Analysis Exoneration',
   107    shouldExtractIngestedInvocationId: true,
   108    shouldExtractPresubmitRunId: false,
   109  }, {
   110    failure: newMockFailure().ingestedInvocationBlocked().dryRun().build(),
   111    filter: 'Without LUCI Analysis Exoneration',
   112    shouldExtractIngestedInvocationId: true,
   113    shouldExtractPresubmitRunId: false,
   114  }, {
   115    failure: newMockFailure().exonerateOccursOnOtherCLs().build(),
   116    filter: 'Without LUCI Analysis Exoneration',
   117    shouldExtractIngestedInvocationId: false,
   118    shouldExtractPresubmitRunId: false,
   119  }, {
   120    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().build(),
   121    filter: 'Without LUCI Analysis Exoneration',
   122    shouldExtractIngestedInvocationId: true,
   123    shouldExtractPresubmitRunId: true,
   124  }, {
   125    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().notPresubmitCritical().build(),
   126    filter: 'Without LUCI Analysis Exoneration',
   127    shouldExtractIngestedInvocationId: true,
   128    shouldExtractPresubmitRunId: false,
   129  }, {
   130    failure: newMockFailure().exonerateNotCritical().build(),
   131    filter: 'Without LUCI Analysis Exoneration',
   132    shouldExtractIngestedInvocationId: false,
   133    shouldExtractPresubmitRunId: false,
   134  }, {
   135    failure: newMockFailure().ingestedInvocationBlocked().exonerateNotCritical().build(),
   136    filter: 'Without LUCI Analysis Exoneration',
   137    shouldExtractIngestedInvocationId: false,
   138    shouldExtractPresubmitRunId: false,
   139  }, {
   140    failure: newMockFailure().build(),
   141    filter: 'Without All Exoneration',
   142    shouldExtractIngestedInvocationId: false,
   143    shouldExtractPresubmitRunId: false,
   144  }, {
   145    failure: newMockFailure().ingestedInvocationBlocked().build(),
   146    filter: 'Without All Exoneration',
   147    shouldExtractIngestedInvocationId: true,
   148    shouldExtractPresubmitRunId: true,
   149  }, {
   150    failure: newMockFailure().exonerateOccursOnOtherCLs().build(),
   151    filter: 'Without All Exoneration',
   152    shouldExtractIngestedInvocationId: false,
   153    shouldExtractPresubmitRunId: false,
   154  }, {
   155    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().build(),
   156    filter: 'Without All Exoneration',
   157    shouldExtractIngestedInvocationId: true,
   158    shouldExtractPresubmitRunId: true,
   159  }, {
   160    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().notPresubmitCritical().build(),
   161    filter: 'Without All Exoneration',
   162    shouldExtractIngestedInvocationId: true,
   163    shouldExtractPresubmitRunId: false,
   164  }, {
   165    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().dryRun().build(),
   166    filter: 'Without All Exoneration',
   167    shouldExtractIngestedInvocationId: true,
   168    shouldExtractPresubmitRunId: false,
   169  }, {
   170    failure: newMockFailure().exonerateNotCritical().build(),
   171    filter: 'Without All Exoneration',
   172    shouldExtractIngestedInvocationId: false,
   173    shouldExtractPresubmitRunId: false,
   174  }, {
   175    failure: newMockFailure().ingestedInvocationBlocked().exonerateNotCritical().build(),
   176    filter: 'Without All Exoneration',
   177    shouldExtractIngestedInvocationId: true,
   178    shouldExtractPresubmitRunId: true,
   179  }, {
   180    failure: newMockFailure().build(),
   181    filter: 'Without Any Retries',
   182    shouldExtractIngestedInvocationId: true,
   183    shouldExtractPresubmitRunId: true,
   184  }, {
   185    failure: newMockFailure().ingestedInvocationBlocked().build(),
   186    filter: 'Without Any Retries',
   187    shouldExtractIngestedInvocationId: true,
   188    shouldExtractPresubmitRunId: true,
   189  }, {
   190    failure: newMockFailure().exonerateOccursOnOtherCLs().build(),
   191    filter: 'Without Any Retries',
   192    shouldExtractIngestedInvocationId: true,
   193    shouldExtractPresubmitRunId: true,
   194  }, {
   195    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().build(),
   196    filter: 'Without Any Retries',
   197    shouldExtractIngestedInvocationId: true,
   198    shouldExtractPresubmitRunId: true,
   199  }, {
   200    failure: newMockFailure().ingestedInvocationBlocked().exonerateOccursOnOtherCLs().notPresubmitCritical().build(),
   201    filter: 'Without Any Retries',
   202    shouldExtractIngestedInvocationId: true,
   203    shouldExtractPresubmitRunId: false,
   204  }])('Extractors with %j', (tc: ExtractorTestCase) => {
   205    it('should return ids in only the cases expected by failure type and impact filter.', () => {
   206      const ingestedInvocationIds = rejectedIngestedInvocationIdsExtractor(impactFilterNamed(tc.filter))(tc.failure);
   207      if (tc.shouldExtractIngestedInvocationId) {
   208        expect(ingestedInvocationIds.size).toBeGreaterThan(0);
   209      } else {
   210        expect(ingestedInvocationIds.size).toBe(0);
   211      }
   212      const presubmitRunIds = rejectedPresubmitRunIdsExtractor(impactFilterNamed(tc.filter))(tc.failure);
   213      if (tc.shouldExtractPresubmitRunId) {
   214        expect(presubmitRunIds.size).toBeGreaterThan(0);
   215      } else {
   216        expect(presubmitRunIds.size).toBe(0);
   217      }
   218    });
   219  });
   220  
   221  describe('groupFailures', () => {
   222    it('should put each failure in a separate group when given unique grouping keys', () => {
   223      const failures = [
   224        newMockFailure().build(),
   225        newMockFailure().build(),
   226        newMockFailure().build(),
   227      ];
   228      let unique = 0;
   229      const groups: FailureGroup[] = groupFailures(failures, () => [{ type: 'variant', key: 'v1', value: '' + unique++ }]);
   230      expect(groups.length).toBe(3);
   231      expect(groups[0].children.length).toBe(1);
   232    });
   233    it('should put each failure in a single group when given a single grouping key', () => {
   234      const failures = [
   235        newMockFailure().build(),
   236        newMockFailure().build(),
   237        newMockFailure().build(),
   238      ];
   239      const groups: FailureGroup[] = groupFailures(failures, () => [{ type: 'variant', key: 'v1', value: 'group1' }]);
   240      expect(groups.length).toBe(1);
   241      expect(groups[0].children.length).toBe(3);
   242    });
   243    it('should put group failures into multiple levels', () => {
   244      const failures = [
   245        newMockFailure().withVariantGroups('v1', 'a').withVariantGroups('v2', 'a').build(),
   246        newMockFailure().withVariantGroups('v1', 'a').withVariantGroups('v2', 'b').build(),
   247        newMockFailure().withVariantGroups('v1', 'b').withVariantGroups('v2', 'a').build(),
   248        newMockFailure().withVariantGroups('v1', 'b').withVariantGroups('v2', 'b').build(),
   249      ];
   250      const groups: FailureGroup[] = groupFailures(failures, (f) => [
   251        { type: 'variant', key: 'v1', value: f.variant?.def['v1'] || '' },
   252        { type: 'variant', key: 'v2', value: f.variant?.def['v2'] || '' },
   253      ]);
   254      expect(groups.length).toBe(2);
   255      expect(groups[0].children.length).toBe(2);
   256      expect(groups[0].commonVariant).toEqual({ def: { 'v1': 'a' } });
   257      expect(groups[1].children.length).toBe(2);
   258      expect(groups[1].commonVariant).toEqual({ def: { 'v1': 'b' } });
   259      expect(groups[0].children[0].children.length).toBe(1);
   260      expect(groups[0].children[0].commonVariant).toEqual({ def: { 'v1': 'a', 'v2': 'a' } });
   261    });
   262  });
   263  
   264  describe('treeDistinctValues', () => {
   265    // A helper to just store the counts to the failures field.
   266    const setFailures = (g: FailureGroup, values: Set<string>) => {
   267      g.failures = values.size;
   268    };
   269    it('should have count of 1 for a valid feature', () => {
   270      const groups = groupFailures([newMockFailure().build()], () => [{ type: 'variant', key: 'v1', value: 'group' }]);
   271  
   272      treeDistinctValues(groups[0], () => new Set(['a']), setFailures);
   273  
   274      expect(groups[0].failures).toBe(1);
   275    });
   276    it('should have count of 0 for an invalid feature', () => {
   277      const groups = groupFailures([newMockFailure().build()], () => [{ type: 'variant', key: 'v1', value: 'group' }]);
   278  
   279      treeDistinctValues(groups[0], () => new Set(), setFailures);
   280  
   281      expect(groups[0].failures).toBe(0);
   282    });
   283  
   284    it('should have count of 1 for two identical features', () => {
   285      const groups = groupFailures([
   286        newMockFailure().build(),
   287        newMockFailure().build(),
   288      ], () => [{ type: 'variant', key: 'v1', value: 'group' }]);
   289  
   290      treeDistinctValues(groups[0], () => new Set(['a']), setFailures);
   291  
   292      expect(groups[0].failures).toBe(1);
   293    });
   294    it('should have count of 2 for two different features', () => {
   295      const groups = groupFailures([
   296        newMockFailure().withTestId('a').build(),
   297        newMockFailure().withTestId('b').build(),
   298      ], () => [{ type: 'variant', key: 'v1', value: 'group' }]);
   299  
   300      treeDistinctValues(groups[0], (f) => f.testId ? new Set([f.testId]) : new Set(), setFailures);
   301  
   302      expect(groups[0].failures).toBe(2);
   303    });
   304    it('should have count of 1 for two identical features in different subgroups', () => {
   305      const groups = groupFailures([
   306        newMockFailure().withTestId('a').withVariantGroups('group', 'a').build(),
   307        newMockFailure().withTestId('a').withVariantGroups('group', 'b').build(),
   308      ], (f) => [{ type: 'variant', key: 'v1', value: 'top' }, { type: 'variant', key: 'v1', value: f.variant?.def['group'] || '' }]);
   309  
   310      treeDistinctValues(groups[0], (f) => f.testId ? new Set([f.testId]) : new Set(), setFailures);
   311  
   312      expect(groups[0].failures).toBe(1);
   313      expect(groups[0].children[0].failures).toBe(1);
   314      expect(groups[0].children[1].failures).toBe(1);
   315    });
   316    it('should have count of 2 for two different features in different subgroups', () => {
   317      const groups = groupFailures([
   318        newMockFailure().withTestId('a').withVariantGroups('group', 'a').build(),
   319        newMockFailure().withTestId('b').withVariantGroups('group', 'b').build(),
   320      ], (f) => [{ type: 'variant', key: 'v1', value: 'top' }, { type: 'variant', key: 'v1', value: f.variant?.def['group'] || '' }]);
   321  
   322      treeDistinctValues(groups[0], (f) => f.testId ? new Set([f.testId]) : new Set(), setFailures);
   323  
   324      expect(groups[0].failures).toBe(2);
   325      expect(groups[0].children[0].failures).toBe(1);
   326      expect(groups[0].children[1].failures).toBe(1);
   327    });
   328  });
   329  
   330  describe('sortFailureGroups', () => {
   331    it('sorts top level groups ascending', () => {
   332      let groups: FailureGroup[] = [
   333        newMockGroup({ type: 'variant', key: 'v1', value: 'c' }).withFailures(3).build(),
   334        newMockGroup({ type: 'variant', key: 'v1', value: 'a' }).withFailures(1).build(),
   335        newMockGroup({ type: 'variant', key: 'v1', value: 'b' }).withFailures(2).build(),
   336      ];
   337  
   338      groups = sortFailureGroups(groups, 'failures', true);
   339  
   340      expect(groups.map((g) => g.key.value)).toEqual(['a', 'b', 'c']);
   341    });
   342    it('sorts top level groups descending', () => {
   343      let groups: FailureGroup[] = [
   344        newMockGroup({ type: 'variant', key: 'v1', value: 'c' }).withFailures(3).build(),
   345        newMockGroup({ type: 'variant', key: 'v1', value: 'a' }).withFailures(1).build(),
   346        newMockGroup({ type: 'variant', key: 'v1', value: 'b' }).withFailures(2).build(),
   347      ];
   348  
   349      groups = sortFailureGroups(groups, 'failures', false);
   350  
   351      expect(groups.map((g) => g.key.value)).toEqual(['c', 'b', 'a']);
   352    });
   353    it('sorts child groups', () => {
   354      let groups: FailureGroup[] = [
   355        newMockGroup({ type: 'variant', key: 'v1', value: 'c' }).withFailures(3).build(),
   356        newMockGroup({ type: 'variant', key: 'v1', value: 'a' }).withFailures(1).withChildren([
   357          newMockGroup({ type: 'variant', key: 'v2', value: 'a3' }).withFailures(3).build(),
   358          newMockGroup({ type: 'variant', key: 'v2', value: 'a2' }).withFailures(2).build(),
   359          newMockGroup({ type: 'variant', key: 'v2', value: 'a1' }).withFailures(1).build(),
   360        ]).build(),
   361        newMockGroup({ type: 'variant', key: 'v1', value: 'b' }).withFailures(2).build(),
   362      ];
   363  
   364      groups = sortFailureGroups(groups, 'failures', true);
   365  
   366      expect(groups.map((g) => g.key.value)).toEqual(['a', 'b', 'c']);
   367      expect(groups[0].children.map((g) => g.key.value)).toEqual(['a1', 'a2', 'a3']);
   368    });
   369    it('sorts on an alternate metric', () => {
   370      let groups: FailureGroup[] = [
   371        newMockGroup({ type: 'variant', key: 'v1', value: 'c' }).withPresubmitRejects(3).build(),
   372        newMockGroup({ type: 'variant', key: 'v1', value: 'a' }).withPresubmitRejects(1).build(),
   373        newMockGroup({ type: 'variant', key: 'v1', value: 'b' }).withPresubmitRejects(2).build(),
   374      ];
   375  
   376      groups = sortFailureGroups(groups, 'presubmitRejects', true);
   377  
   378      expect(groups.map((g) => g.key.value)).toEqual(['a', 'b', 'c']);
   379    });
   380  });