go.fuchsia.dev/infra@v0.0.0-20240507153436-9b593402251b/cmd/autocorrelator/check_ci_test.go (about) 1 // Copyright 2021 The Fuchsia Authors. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 package main 6 7 import ( 8 "testing" 9 10 buildbucketpb "go.chromium.org/luci/buildbucket/proto" 11 "go.chromium.org/luci/common/proto/git" 12 "go.fuchsia.dev/infra/cmd/autocorrelator/findings" 13 14 "github.com/google/go-cmp/cmp" 15 ) 16 17 func TestCheckCI(t *testing.T) { 18 t.Parallel() 19 20 tests := []struct { 21 name string 22 log []*git.Commit 23 builds []*buildbucketpb.Build 24 status buildbucketpb.Status 25 summaryMarkdown string 26 expected *findings.SummarySimilarity 27 }{ 28 { 29 name: "ci is green", 30 log: []*git.Commit{ 31 { 32 Id: "D", 33 }, 34 { 35 Id: "C", 36 }, 37 { 38 Id: "B", 39 }, 40 { 41 Id: "A", 42 }, 43 }, 44 // SUCCESS > FAILURE case, where we are looking for FAILURE. 45 builds: []*buildbucketpb.Build{ 46 // This build's status is SUCCESS. We should return a finding 47 // as soon as we see this, indicating that CI does not appear 48 // broken. 49 { 50 Id: int64(999999), 51 Input: &buildbucketpb.Build_Input{ 52 GitilesCommit: &buildbucketpb.GitilesCommit{ 53 Id: "C", 54 }, 55 }, 56 Status: buildbucketpb.Status_SUCCESS, 57 SummaryMarkdown: "", 58 }, 59 // Even though this build's status is FAILURE, it should be 60 // ignored, since the above build should have been caught first. 61 { 62 Id: int64(999998), 63 Input: &buildbucketpb.Build_Input{ 64 GitilesCommit: &buildbucketpb.GitilesCommit{ 65 Id: "A", 66 }, 67 }, 68 Status: buildbucketpb.Status_FAILURE, 69 SummaryMarkdown: "ERROR", 70 }, 71 }, 72 status: buildbucketpb.Status_FAILURE, 73 summaryMarkdown: "ERROR", 74 expected: &findings.SummarySimilarity{ 75 Score: 0.0, 76 BuildId: "999999", 77 // The caught build has commit ID C which is one commit away 78 // from the front of the log. 79 CommitDist: 1, 80 IsGreen: true, 81 }, 82 }, 83 { 84 name: "ci is red", 85 log: []*git.Commit{ 86 { 87 Id: "D", 88 }, 89 { 90 Id: "C", 91 }, 92 { 93 Id: "B", 94 }, 95 { 96 Id: "A", 97 }, 98 }, 99 // Simple FAILURE case, where we are looking for FAILURE. 100 builds: []*buildbucketpb.Build{ 101 // This build's commit ID is not in the log, and should be 102 // ignored, even though the build's status matches the input 103 // status. 104 { 105 Id: int64(999999), 106 Input: &buildbucketpb.Build_Input{ 107 GitilesCommit: &buildbucketpb.GitilesCommit{ 108 Id: "E", 109 }, 110 }, 111 Status: buildbucketpb.Status_FAILURE, 112 SummaryMarkdown: "ERROR", 113 }, 114 // This build's status is FAILURE, which matches the input 115 // status. We should catch this build. 116 { 117 Id: int64(999998), 118 Input: &buildbucketpb.Build_Input{ 119 GitilesCommit: &buildbucketpb.GitilesCommit{ 120 Id: "C", 121 }, 122 }, 123 Status: buildbucketpb.Status_FAILURE, 124 SummaryMarkdown: "ERROR", 125 }, 126 // Even though this build's status is FAILURE, it should be 127 // ignored, since the above build should have been caught first. 128 { 129 Id: int64(999997), 130 Input: &buildbucketpb.Build_Input{ 131 GitilesCommit: &buildbucketpb.GitilesCommit{ 132 Id: "A", 133 }, 134 }, 135 Status: buildbucketpb.Status_FAILURE, 136 SummaryMarkdown: "ERROR", 137 }, 138 }, 139 status: buildbucketpb.Status_FAILURE, 140 summaryMarkdown: "ERROR", 141 expected: &findings.SummarySimilarity{ 142 Score: 1.0, 143 BuildId: "999998", 144 // The caught build has commit ID C which is one commit away 145 // from the front of the log. 146 CommitDist: 1, 147 IsGreen: false, 148 }, 149 }, 150 { 151 name: "look for purple", 152 log: []*git.Commit{ 153 { 154 Id: "D", 155 }, 156 { 157 Id: "C", 158 }, 159 { 160 Id: "B", 161 }, 162 { 163 Id: "A", 164 }, 165 }, 166 // FAILURE > INFRA_FAILURE case, where we are looking for 167 // INFRA_FAILURE. 168 builds: []*buildbucketpb.Build{ 169 // This build's status is FAILURE, which does not match the 170 // input status, and should be ignored. 171 { 172 Id: int64(999999), 173 Input: &buildbucketpb.Build_Input{ 174 GitilesCommit: &buildbucketpb.GitilesCommit{ 175 Id: "C", 176 }, 177 }, 178 Status: buildbucketpb.Status_FAILURE, 179 SummaryMarkdown: "ERROR", 180 }, 181 // This build's status is INFRA_FAILURE, which does match the 182 // input status, and should be caught. 183 { 184 Id: int64(999998), 185 Input: &buildbucketpb.Build_Input{ 186 GitilesCommit: &buildbucketpb.GitilesCommit{ 187 Id: "B", 188 }, 189 }, 190 Status: buildbucketpb.Status_INFRA_FAILURE, 191 SummaryMarkdown: "ERROR: checkout failed", 192 }, 193 }, 194 status: buildbucketpb.Status_INFRA_FAILURE, 195 summaryMarkdown: "ERROR: checkout failed", 196 expected: &findings.SummarySimilarity{ 197 Score: 1.0, 198 BuildId: "999998", 199 // The caught build has commit ID B which is three commits away 200 // from the front of the log. 201 CommitDist: 2, 202 IsGreen: false, 203 }, 204 }, 205 { 206 name: "look for red", 207 log: []*git.Commit{ 208 { 209 Id: "D", 210 }, 211 { 212 Id: "C", 213 }, 214 { 215 Id: "B", 216 }, 217 { 218 Id: "A", 219 }, 220 }, 221 // INFRA_FAILURE > FAILURE case, where we are looking for FAILURE. 222 builds: []*buildbucketpb.Build{ 223 // This build's status is INFRA_FAILURE, which does not match 224 // the input status, and should be ignored. 225 { 226 Id: int64(999999), 227 Input: &buildbucketpb.Build_Input{ 228 GitilesCommit: &buildbucketpb.GitilesCommit{ 229 Id: "C", 230 }, 231 }, 232 Status: buildbucketpb.Status_INFRA_FAILURE, 233 SummaryMarkdown: "ERROR: checkout failed", 234 }, 235 // This build's status is FAILURE, which does match the input 236 // status, and should be caught. 237 { 238 Id: int64(999998), 239 Input: &buildbucketpb.Build_Input{ 240 GitilesCommit: &buildbucketpb.GitilesCommit{ 241 Id: "A", 242 }, 243 }, 244 Status: buildbucketpb.Status_FAILURE, 245 SummaryMarkdown: "ERROR", 246 }, 247 }, 248 status: buildbucketpb.Status_FAILURE, 249 summaryMarkdown: "ERROR", 250 expected: &findings.SummarySimilarity{ 251 Score: 1.0, 252 BuildId: "999998", 253 // The caught build has commit ID A which is three commits away 254 // from the front of the log. 255 CommitDist: 3, 256 IsGreen: false, 257 }, 258 }, 259 // Build exhaustion, where we are looking for FAILURE. 260 { 261 name: "build exhaustion", 262 log: []*git.Commit{ 263 { 264 Id: "D", 265 }, 266 { 267 Id: "C", 268 }, 269 { 270 Id: "B", 271 }, 272 { 273 Id: "A", 274 }, 275 }, 276 builds: []*buildbucketpb.Build{ 277 // This build's status is INFRA_FAILURE, which does not match 278 // the input status, and should be ignored. 279 { 280 Id: int64(999999), 281 Input: &buildbucketpb.Build_Input{ 282 GitilesCommit: &buildbucketpb.GitilesCommit{ 283 Id: "C", 284 }, 285 }, 286 Status: buildbucketpb.Status_INFRA_FAILURE, 287 SummaryMarkdown: "ERROR: checkout failed", 288 }, 289 }, 290 status: buildbucketpb.Status_FAILURE, 291 summaryMarkdown: "ERROR", 292 // We should have exhausted the list of builds without having found 293 // anything. 294 expected: nil, 295 }, 296 } 297 298 for _, test := range tests { 299 t.Run(test.name, func(t *testing.T) { 300 comparator := mockComparator{} 301 ss := checkCI(test.log, test.builds, test.status, test.summaryMarkdown, comparator) 302 if diff := cmp.Diff(test.expected, ss); diff != "" { 303 t.Fatalf("different (-want +got):\n%s", diff) 304 } 305 }) 306 } 307 }