github.com/GoogleCloudPlatform/testgrid@v0.0.174/pkg/summarizer/analyzers/flipanalyzer_test.go (about)

     1  /*
     2  Copyright 2020 The TestGrid Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package analyzers
    18  
    19  import (
    20  	"testing"
    21  
    22  	summarypb "github.com/GoogleCloudPlatform/testgrid/pb/summary"
    23  	"github.com/GoogleCloudPlatform/testgrid/pkg/summarizer/common"
    24  	"github.com/google/go-cmp/cmp"
    25  	"google.golang.org/protobuf/testing/protocmp"
    26  )
    27  
    28  func TestGetFlakinessFlip(t *testing.T) {
    29  	cases := []struct {
    30  		name                   string
    31  		metrics                []*common.GridMetrics
    32  		relevantFilteredStatus map[string][]StatusCategory
    33  		minRuns                int
    34  		startDate              int
    35  		endDate                int
    36  		tab                    string
    37  		expected               *summarypb.HealthinessInfo
    38  	}{
    39  		{
    40  			name: "case where naive and enhanced flakiness disagree",
    41  			metrics: []*common.GridMetrics{
    42  				{
    43  					Name:             "//test1 - [env1]",
    44  					Passed:           3,
    45  					Failed:           3,
    46  					FlakyCount:       0,
    47  					AverageFlakiness: 50.0,
    48  					FailedInfraCount: 1,
    49  				},
    50  			},
    51  			relevantFilteredStatus: map[string][]StatusCategory{
    52  				"//test1 - [env1]": {
    53  					StatusFail, StatusFail, StatusFail, StatusPass, StatusPass, StatusPass,
    54  				},
    55  			},
    56  			minRuns:   -1,
    57  			startDate: 0,
    58  			endDate:   2,
    59  			tab:       "tab1",
    60  			expected: &summarypb.HealthinessInfo{
    61  				Start: createTimestamp(0),
    62  				End:   createTimestamp(2),
    63  				Tests: []*summarypb.TestInfo{
    64  					{
    65  						DisplayName:        "//test1 - [env1]",
    66  						TotalNonInfraRuns:  6,
    67  						TotalRunsWithInfra: 7,
    68  						PassedNonInfraRuns: 3,
    69  						FailedNonInfraRuns: 3,
    70  						FailedInfraRuns:    1,
    71  						Flakiness:          0,
    72  					},
    73  				},
    74  				AverageFlakiness: 0,
    75  			},
    76  		},
    77  	}
    78  
    79  	for _, tc := range cases {
    80  		t.Run(tc.name, func(t *testing.T) {
    81  			analyzer := FlipAnalyzer{
    82  				RelevantStatus: tc.relevantFilteredStatus,
    83  			}
    84  			actual := analyzer.GetFlakiness(tc.metrics, tc.minRuns, tc.startDate, tc.endDate, tc.tab)
    85  			if diff := cmp.Diff(tc.expected, actual, protocmp.Transform()); diff != "" {
    86  				t.Errorf("\nGetFlakiness produced unexpected diff (-want +got): %s", diff)
    87  			}
    88  		})
    89  	}
    90  }
    91  
    92  func almostEqual(a, b float32) bool {
    93  	return (a-b < .01) && (b-a < .01)
    94  }
    95  
    96  func TestCalculateFlipFlakiness(t *testing.T) {
    97  	p := StatusPass
    98  	f := StatusFail
    99  	fl := StatusFlaky
   100  	cases := []struct {
   101  		name     string
   102  		results  []StatusCategory
   103  		expected float32
   104  	}{
   105  		{
   106  			name:     "Empty List",
   107  			results:  []StatusCategory{},
   108  			expected: 0,
   109  		},
   110  		{
   111  			name:     "All passing",
   112  			results:  []StatusCategory{p, p, p},
   113  			expected: 0,
   114  		},
   115  		{
   116  			name:     "Multiple flakes",
   117  			results:  []StatusCategory{p, f, p, p, fl, p, p, p, p, f},
   118  			expected: 30,
   119  		},
   120  		{
   121  			name:     "Short run",
   122  			results:  []StatusCategory{p, f, f, p, p, p, p, f, p, p},
   123  			expected: 20,
   124  		},
   125  		{
   126  			name:     "Long run",
   127  			results:  []StatusCategory{p, f, f, f, f, f, p, f, p, f},
   128  			expected: 40,
   129  		},
   130  		{
   131  			name:     "Long run interupted by flakes",
   132  			results:  []StatusCategory{p, f, f, fl, f, f, p, f, p, f},
   133  			expected: 50,
   134  		},
   135  	}
   136  
   137  	for _, tc := range cases {
   138  		t.Run(tc.name, func(t *testing.T) {
   139  			if actual := calculateFlipFlakiness(tc.results); !almostEqual(actual, tc.expected) {
   140  				t.Errorf("enhancedFlakiness(%+v)=%v; expected %v", tc.results, actual, tc.expected)
   141  			}
   142  		})
   143  	}
   144  }