github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/tools/test_monitor/level3/process_summary3_results_test.go (about)

     1  package main
     2  
     3  import (
     4  	"encoding/json"
     5  	"os"
     6  	"path/filepath"
     7  	"testing"
     8  
     9  	"github.com/onflow/flow-go/tools/test_monitor/common"
    10  
    11  	"github.com/stretchr/testify/require"
    12  )
    13  
    14  func TestGenerateLevel3Summary(t *testing.T) {
    15  	testDataMap := map[string]string{
    16  		"1 failure the rest pass":                     "test1-1package-1failure",
    17  		"1 exception, no other tests":                 "test2-1-exception",
    18  		"many exceptions":                             "test3-multi-exceptions",
    19  		"many failures, many passes":                  "test4-multi-failures",
    20  		"many failures, many passes, many exceptions": "test5-multi-durations",
    21  		"many failures - cap failures":                "test6-multi-failures-cap",
    22  		"many durations - cap durations":              "test7-multi-durations-cap",
    23  	}
    24  
    25  	for k, testDir := range testDataMap {
    26  		t.Run(k, func(t *testing.T) {
    27  			runGenerateLevel3Summary(t, testDir)
    28  		})
    29  	}
    30  
    31  }
    32  
    33  const testDataDir = "../testdata/summary3"
    34  
    35  func runGenerateLevel3Summary(t *testing.T, testDir string) {
    36  
    37  	testDataBaseDir := filepath.Join(testDataDir, testDir)
    38  	inputTestDataPath := filepath.Join(testDataBaseDir, "input", testDir+".json")
    39  	expectedOutputTestDataPath := filepath.Join(testDataBaseDir, "expected-output", testDir+".json")
    40  
    41  	// **************************************************************
    42  	actualTestSummary3 := generateLevel3Summary(inputTestDataPath, filepath.Join(testDataBaseDir, "input"))
    43  	// **************************************************************
    44  
    45  	// read in expected summary level 3
    46  	var expectedTestSummary3 common.Level3Summary
    47  	expectedTestSummary3JsonBytes, err := os.ReadFile(expectedOutputTestDataPath)
    48  	require.Nil(t, err)
    49  	require.NotEmpty(t, expectedTestSummary3JsonBytes)
    50  	err = json.Unmarshal(expectedTestSummary3JsonBytes, &expectedTestSummary3)
    51  	require.Nil(t, err)
    52  
    53  	// check all details of test summary level 2 between expected and actual
    54  
    55  	// check # of exceptions, failures and longest durations is the same for expected vs actual
    56  	require.Equal(t, len(expectedTestSummary3.Exceptions), len(actualTestSummary3.Exceptions))
    57  
    58  	require.Equal(t, len(expectedTestSummary3.MostFailures), len(actualTestSummary3.MostFailures))
    59  	require.Equal(t, expectedTestSummary3.MostFailuresTotal, actualTestSummary3.MostFailuresTotal)
    60  
    61  	require.Equal(t, len(expectedTestSummary3.LongestRunning), len(actualTestSummary3.LongestRunning))
    62  	require.Equal(t, expectedTestSummary3.LongestRunningTotal, actualTestSummary3.LongestRunningTotal)
    63  
    64  	// check exception, failure and duration lists are the same for expected vs actual
    65  	for ExceptionsIndex := range expectedTestSummary3.Exceptions {
    66  		common.AssertLevel2TestResults(t, expectedTestSummary3.Exceptions[ExceptionsIndex], actualTestSummary3.Exceptions[ExceptionsIndex])
    67  	}
    68  
    69  	for failuresIndex := range expectedTestSummary3.MostFailures {
    70  		common.AssertLevel2TestResults(t, expectedTestSummary3.MostFailures[failuresIndex], actualTestSummary3.MostFailures[failuresIndex])
    71  	}
    72  
    73  	for durationIndex := range expectedTestSummary3.LongestRunning {
    74  		common.AssertLevel2TestResults(t, expectedTestSummary3.LongestRunning[durationIndex], actualTestSummary3.LongestRunning[durationIndex])
    75  	}
    76  
    77  	require.Equal(t, expectedTestSummary3, actualTestSummary3)
    78  }
    79  
    80  // test that script panics when supplied file path is invalid (can't find file)
    81  func TestGenerateLevel3Summary_Panic_WrongPath(t *testing.T) {
    82  	require.PanicsWithValue(t, "error reading level 2 json: open foobar: no such file or directory",
    83  		func() {
    84  			generateLevel3Summary("foobar", ".")
    85  		})
    86  }
    87  
    88  // test that script panics when supplied file is not valid level 2 format
    89  func TestGenerateLevel3Summary_Panic_WrongFormat(t *testing.T) {
    90  	require.PanicsWithValue(t, "invalid summary 2 file - no test results found",
    91  		func() {
    92  			// supplied file is level 3 file, not level 2 - this should cause a panic
    93  			generateLevel3Summary(filepath.Join(testDataDir, "test1-1package-1failure/expected-output/test1-1package-1failure.json"), ".")
    94  		})
    95  }