github.com/GoogleCloudPlatform/testgrid@v0.0.174/pkg/summarizer/analyzers/flipanalyzer.go (about)

     1  /*
     2  Copyright 2020 The TestGrid Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package analyzers represents ways to analyze healthiness and flakiness of tests
    18  package analyzers
    19  
    20  import (
    21  	summarypb "github.com/GoogleCloudPlatform/testgrid/pb/summary"
    22  	"github.com/GoogleCloudPlatform/testgrid/pkg/summarizer/common"
    23  )
    24  
    25  const analyzerName = "flipanalyzer"
    26  
    27  // FlipAnalyzer implements functions that calculate flakiness as a ratio of failed tests to total tests
    28  type FlipAnalyzer struct {
    29  	RelevantStatus map[string][]StatusCategory
    30  }
    31  
    32  // StatusCategory is a simiplified status that allows only "Pass", "Fail", and "Flaky"
    33  type StatusCategory int32
    34  
    35  // StatusPass, StatusFail, and StatusFlaky are the status categories this analyzer works with.
    36  const (
    37  	StatusPass StatusCategory = iota
    38  	StatusFail
    39  	StatusFlaky
    40  )
    41  
    42  // GetFlakiness returns a HealthinessInfo message
    43  func (ea *FlipAnalyzer) GetFlakiness(gridMetrics []*common.GridMetrics, minRuns int, startDate int, endDate int, tab string) *summarypb.HealthinessInfo {
    44  	// Delegate to a BaseAnalyzer and change the flakiness scores.
    45  	// (And average flakiness)
    46  	var ba BaseAnalyzer
    47  	healthinessInfo := ba.GetFlakiness(gridMetrics, minRuns, startDate, endDate, tab)
    48  	var averageFlakiness float32
    49  	for _, test := range healthinessInfo.Tests {
    50  		test.Flakiness = calculateFlipFlakiness(ea.RelevantStatus[test.DisplayName])
    51  		averageFlakiness += test.Flakiness
    52  	}
    53  	if len(healthinessInfo.Tests) == 0 {
    54  		healthinessInfo.AverageFlakiness = 0
    55  	} else {
    56  		healthinessInfo.AverageFlakiness = averageFlakiness / float32(len(healthinessInfo.Tests))
    57  	}
    58  	return healthinessInfo
    59  }
    60  
    61  const ignoreFailuresInARow = 3
    62  
    63  func consecutiveFailures(statuses []StatusCategory, i int) int {
    64  	var result int
    65  	for i < len(statuses) && statuses[i] == StatusFail {
    66  		result++
    67  		i++
    68  	}
    69  
    70  	return result
    71  }
    72  
    73  // calculateFlipFlakiness gets a calculation of flakiness based on number of flips to failing rather than number of failures
    74  // statuses should have already filtered to the correct time horizon and removed infra failures
    75  // Returns a percentage between 0 and 100
    76  func calculateFlipFlakiness(statuses []StatusCategory) float32 {
    77  	var flips int
    78  	var considered int
    79  	lastPassing := true // No flakes if we pass 100%
    80  	var i int
    81  	for i < len(statuses) {
    82  		cf := consecutiveFailures(statuses, i)
    83  		if cf >= ignoreFailuresInARow {
    84  			// Ignore the run of failures
    85  			i += cf
    86  			if i >= len(statuses) {
    87  				break
    88  			}
    89  		}
    90  		s := statuses[i]
    91  		considered++
    92  		if s == StatusPass {
    93  			lastPassing = true
    94  		} else if s == StatusFlaky {
    95  			// Consider this as always a flip (because there was a flip involved), but it did pass.
    96  			flips++
    97  			lastPassing = true
    98  		} else {
    99  			// Failing
   100  			if lastPassing {
   101  				flips++
   102  			}
   103  			lastPassing = false
   104  		}
   105  		i++
   106  	}
   107  	if considered == 0 {
   108  		return 0
   109  	}
   110  	return 100 * float32(flips) / float32(considered)
   111  
   112  }