github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/shared/run_diff_test.go (about) 1 // +build small 2 3 // Copyright 2017 The WPT Dashboard Project. All rights reserved. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE file. 6 7 package shared_test 8 9 import ( 10 "context" 11 "encoding/json" 12 "strings" 13 "testing" 14 15 mapset "github.com/deckarep/golang-set" 16 "github.com/golang/mock/gomock" 17 "github.com/stretchr/testify/assert" 18 "github.com/web-platform-tests/wpt.fyi/shared" 19 "github.com/web-platform-tests/wpt.fyi/shared/sharedtest" 20 ) 21 22 const mockTestPath = "/mock/path.html" 23 24 func allDifferences() shared.DiffFilterParam { 25 return shared.DiffFilterParam{ 26 Added: true, 27 Deleted: true, 28 Changed: true, 29 Unchanged: true, 30 } 31 } 32 33 func TestDiffResults_NoDifference(t *testing.T) { 34 assertNoDeltaDifferences(t, []int{0, 1}, []int{0, 1}) 35 assertNoDeltaDifferences(t, []int{3, 4}, []int{3, 4}) 36 } 37 38 func TestDiffResults_Difference(t *testing.T) { 39 // One test now passing 40 assertDelta(t, []int{0, 1}, []int{1, 1}, []int{1, 0, 0}) 41 42 // One test now failing 43 assertDelta(t, []int{1, 1}, []int{0, 1}, []int{0, 1, 0}) 44 45 // Two tests, one now failing 46 assertDelta(t, []int{2, 2}, []int{1, 2}, []int{0, 1, 0}) 47 48 // Three tests, two now passing 49 assertDelta(t, []int{1, 3}, []int{3, 3}, []int{2, 0, 0}) 50 } 51 52 func TestDiffResults_Added(t *testing.T) { 53 // One new test, all passing 54 assertDelta(t, []int{1, 1}, []int{2, 2}, []int{1, 0, 1}) 55 56 // One new test, all failing 57 assertDelta(t, []int{0, 1}, []int{0, 2}, []int{0, 1, 1}) 58 59 // One new test, new test passing 60 assertDelta(t, []int{0, 1}, []int{1, 2}, []int{1, 0, 1}) 61 62 // One new test, new test failing 63 assertDelta(t, []int{1, 1}, []int{1, 2}, []int{0, 1, 1}) 64 65 // Added, but it was a rename. 66 renames := map[string]string{"/foo.html": "/bar.html"} 67 rBefore := shared.ResultsSummary{"/foo.html": []int{1, 1}} 68 rAfter := shared.ResultsSummary{"/bar.html": []int{1, 1}} 69 assert.Equal( 70 t, 71 map[string]shared.TestDiff{"/bar.html": {0, 0, 0}}, 72 shared.GetResultsDiff(rBefore, rAfter, allDifferences(), nil, renames)) 73 } 74 75 func TestDiffResults_Removed(t *testing.T) { 76 // One removed test, all passing 77 assertDelta(t, []int{2, 2}, []int{1, 1}, []int{0, 0, -1}) 78 79 // One removed test, all failing 80 assertDelta(t, []int{0, 2}, []int{0, 1}, []int{0, 0, -1}) 81 82 // One removed test, deleted test passing 83 assertDelta(t, []int{1, 2}, []int{0, 1}, []int{0, 0, -1}) 84 85 // One removed test, deleted test failing 86 assertDelta(t, []int{1, 2}, []int{1, 1}, []int{0, 0, -1}) 87 } 88 89 func TestDiffResults_Filtered(t *testing.T) { 90 changedFilter := shared.DiffFilterParam{Changed: true} 91 addedFilter := shared.DiffFilterParam{Added: true} 92 deletedFilter := shared.DiffFilterParam{Deleted: true} 93 const removedPath = "/mock/removed.html" 94 const changedPath = "/mock/changed.html" 95 const addedPath = "/mock/added.html" 96 97 before := shared.ResultsSummary{ 98 removedPath: {1, 2}, 99 changedPath: {2, 5}, 100 } 101 after := shared.ResultsSummary{ 102 changedPath: {3, 5}, 103 addedPath: {1, 3}, 104 } 105 assert.Equal(t, map[string]shared.TestDiff{changedPath: {1, 0, 0}}, shared.GetResultsDiff(before, after, changedFilter, nil, nil)) 106 assert.Equal(t, map[string]shared.TestDiff{addedPath: {1, 2, 3}}, shared.GetResultsDiff(before, after, addedFilter, nil, nil)) 107 assert.Equal(t, map[string]shared.TestDiff{removedPath: {0, 0, -2}}, shared.GetResultsDiff(before, after, deletedFilter, nil, nil)) 108 109 // Test filtering by each /, /mock/, and /mock/path.html 110 pieces := strings.SplitAfter(mockTestPath, "/") 111 for i := 1; i < len(pieces); i++ { 112 paths := mapset.NewSet(strings.Join(pieces[:i], "")) 113 filter := shared.DiffFilterParam{Changed: true} 114 assertDeltaWithFilter(t, []int{1, 3}, []int{2, 4}, []int{1, 0, 1}, filter, paths) 115 } 116 117 // Filter where none match 118 rBefore, rAfter := getDeltaResultsMaps([]int{0, 5}, []int{5, 5}) 119 filter := shared.DiffFilterParam{Changed: true} 120 paths := mapset.NewSet("/different/path/") 121 assert.Empty(t, shared.GetResultsDiff(rBefore, rAfter, filter, paths, nil)) 122 123 // Filter where one matches 124 mockPath1, mockPath2 := "/mock/path-1.html", "/mock/path-2.html" 125 rBefore = shared.ResultsSummary{ 126 mockPath1: {0, 1}, 127 mockPath2: {0, 1}, 128 } 129 rAfter = shared.ResultsSummary{ 130 mockPath1: {2, 2}, 131 mockPath2: {2, 2}, 132 } 133 delta := shared.GetResultsDiff(rBefore, rAfter, filter, mapset.NewSet(mockPath1), nil) 134 assert.NotContains(t, delta, mockPath2) 135 assert.Contains(t, delta, mockPath1) 136 assert.Equal(t, shared.TestDiff{2, 0, 1}, delta[mockPath1]) 137 } 138 139 func assertNoDeltaDifferences(t *testing.T, before []int, after []int) { 140 assertNoDeltaDifferencesWithFilter(t, before, after, shared.DiffFilterParam{Added: true, Deleted: true, Changed: true}) 141 } 142 143 func assertNoDeltaDifferencesWithFilter(t *testing.T, before []int, after []int, filter shared.DiffFilterParam) { 144 rBefore, rAfter := getDeltaResultsMaps(before, after) 145 assert.Equal(t, map[string]shared.TestDiff{}, shared.GetResultsDiff(rBefore, rAfter, filter, nil, nil)) 146 } 147 148 func assertDelta(t *testing.T, before []int, after []int, delta []int) { 149 assertDeltaWithFilter(t, before, after, delta, shared.DiffFilterParam{Added: true, Deleted: true, Changed: true}, nil) 150 } 151 152 func assertDeltaWithFilter(t *testing.T, before []int, after []int, delta []int, filter shared.DiffFilterParam, paths mapset.Set) { 153 rBefore, rAfter := getDeltaResultsMaps(before, after) 154 assert.Equal( 155 t, 156 map[string]shared.TestDiff{mockTestPath: delta}, 157 shared.GetResultsDiff(rBefore, rAfter, filter, paths, nil)) 158 } 159 160 func getDeltaResultsMaps(before []int, after []int) (shared.ResultsSummary, shared.ResultsSummary) { 161 return shared.ResultsSummary{mockTestPath: before}, 162 shared.ResultsSummary{mockTestPath: after} 163 } 164 165 func TestRegressions(t *testing.T) { 166 // Note: shared.TestDiff items are {passing, regressed, total-delta}. 167 regressed := shared.TestDiff{0, 1, 0} 168 assert.Equal(t, 1, regressed.Regressions()) 169 diff := shared.ResultsDiff{"/foo.html": regressed} 170 regressions := diff.Regressions() 171 assert.Equal(t, 1, regressions.Cardinality()) 172 assert.True(t, regressions.Contains("/foo.html")) 173 174 newlyPassed := shared.TestDiff{1, 0, 1} 175 assert.Equal(t, 0, newlyPassed.Regressions()) 176 diff = shared.ResultsDiff{"/bar.html": newlyPassed} 177 regressions = diff.Regressions() 178 assert.Equal(t, 0, regressions.Cardinality()) 179 assert.False(t, regressions.Contains("/bar.html")) 180 181 // A reduction in test-count is treated as though that test regressed, 182 // in spite of there being zero newly-failing tests. 183 droppedTests := shared.TestDiff{0, 0, -2} 184 assert.Equal(t, 0, droppedTests.Regressions()) 185 diff = shared.ResultsDiff{"/baz.html": droppedTests} 186 regressions = diff.Regressions() 187 assert.Equal(t, 1, regressions.Cardinality()) 188 assert.True(t, regressions.Contains("/baz.html")) 189 } 190 191 func TestRunDiffFromSearchResponse(t *testing.T) { 192 body := []byte(`{ 193 "runs": [{ 194 "id": 203760020, 195 "browser_name": "safari", 196 "browser_version": "82 preview", 197 "os_name": "mac", 198 "os_version": "10.13", 199 "revision": "b5d4599280", 200 "full_revision_hash": "b5d4599280363dc4e4e6a87f3706f0edce5bbdb6", 201 "results_url": "https://storage.googleapis.com/wptd-staging/b5d4599280363dc4e4e6a87f3706f0edce5bbdb6/safari-82_preview-mac-10.13-be2f6871ef-summary.json.gz", 202 "created_at": "2019-06-18T17:30:23.755776Z", 203 "time_start": "2019-06-18T17:27:54.716Z", 204 "time_end": "2019-06-18T17:29:39.042Z", 205 "raw_results_url": "https://storage.googleapis.com/wptd-results-staging/b5d4599280363dc4e4e6a87f3706f0edce5bbdb6/safari-82_preview-mac-10.13-be2f6871ef/report.json", 206 "labels": ["azure", "experimental", "pr_base", "preview", "safari"] 207 }, { 208 "id": 207750011, 209 "browser_name": "safari", 210 "browser_version": "82 preview", 211 "os_name": "mac", 212 "os_version": "10.13", 213 "revision": "b5d4599280", 214 "full_revision_hash": "b5d4599280363dc4e4e6a87f3706f0edce5bbdb6", 215 "results_url": "https://storage.googleapis.com/wptd-staging/b5d4599280363dc4e4e6a87f3706f0edce5bbdb6/safari-82_preview-mac-10.13-b58260d2de-summary.json.gz", 216 "created_at": "2019-06-18T17:33:37.68543Z", 217 "time_start": "2019-06-18T17:30:57.578Z", 218 "time_end": "2019-06-18T17:32:50.741Z", 219 "raw_results_url": "https://storage.googleapis.com/wptd-results-staging/b5d4599280363dc4e4e6a87f3706f0edce5bbdb6/safari-82_preview-mac-10.13-b58260d2de/report.json", 220 "labels": ["azure", "experimental", "pr_head", "preview", "safari"] 221 }], 222 "results": [ 223 { 224 "test":"/media-source/idlharness.any.worker.html", 225 "legacy_status":[{"passes":1,"total":2},{"passes":1,"total":2}] 226 }, 227 { 228 "test": "/pointerevents/idlharness.window.html", 229 "legacy_status": [{ "passes": 4, "total": 5 }, { "passes": 76, "total": 84 }], 230 "diff": [72, 8, 0] 231 } 232 ] 233 }`) 234 var scDiff shared.SearchResponse 235 json.Unmarshal(body, &scDiff) 236 237 mockCtrl := gomock.NewController(t) 238 defer mockCtrl.Finish() 239 aeAPI := sharedtest.NewMockAppEngineAPI(mockCtrl) 240 aeAPI.EXPECT().Context().AnyTimes().Return(context.Background()) 241 aeAPI.EXPECT().IsFeatureEnabled("diffRenames").Return(false) 242 243 diff, err := shared.RunDiffFromSearchResponse(aeAPI, shared.TestRun{}, shared.TestRun{}, scDiff) 244 assert.Nil(t, err) 245 assert.Equal(t, 1, diff.Differences.Regressions().Cardinality()) 246 assert.Equal(t, 4, diff.BeforeSummary["/pointerevents/idlharness.window.html"][0]) 247 assert.Equal(t, 5, diff.BeforeSummary["/pointerevents/idlharness.window.html"][1]) 248 assert.Equal(t, 76, diff.AfterSummary["/pointerevents/idlharness.window.html"][0]) 249 assert.Equal(t, 84, diff.AfterSummary["/pointerevents/idlharness.window.html"][1]) 250 }