github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/internal/ci/workflows.cue (about) 1 // Copyright 2021 The CUE Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package ci 16 17 import ( 18 "github.com/SchemaStore/schemastore/src/schemas/json" 19 encjson "encoding/json" 20 "strconv" 21 ) 22 23 workflowsDir: *"./" | string @tag(workflowsDir) 24 25 _#masterBranch: "master" 26 _#releaseTagPattern: "v*" 27 28 workflows: [...{file: string, schema: (json.#Workflow & {})}] 29 workflows: [ 30 { 31 file: "test.yml" 32 schema: test 33 }, 34 { 35 file: "repository_dispatch.yml" 36 schema: repository_dispatch 37 }, 38 { 39 file: "release.yml" 40 schema: release 41 }, 42 { 43 file: "tip_triggers.yml" 44 schema: tip_triggers 45 }, 46 ] 47 48 test: _#bashWorkflow & { 49 50 name: "Test" 51 on: { 52 push: { 53 branches: ["**"] // any branch (including '/' namespaced branches) 54 "tags-ignore": [_#releaseTagPattern] 55 } 56 pull_request: {} 57 } 58 59 jobs: { 60 start: { 61 "runs-on": _#linuxMachine 62 steps: [...(_ & {if: "${{ \(_#isCLCITestBranch) }}"})] 63 steps: [ 64 _#writeNetrcFile, 65 _#startCLBuild, 66 ] 67 } 68 test: { 69 needs: "start" 70 strategy: _#testStrategy 71 "runs-on": "${{ matrix.os }}" 72 steps: [ 73 _#writeNetrcFile, 74 _#installGo, 75 _#checkoutCode, 76 _#cacheGoModules, 77 _#setGoBuildTags & { 78 _#tags: "long" 79 if: "${{ \(_#isMaster) }}" 80 }, 81 _#goGenerate, 82 _#goTest, 83 _#goTestRace & { 84 if: "${{ matrix.go-version == '\(_#latestStableGo)' && matrix.os == '\(_#linuxMachine)' }}" 85 }, 86 _#goReleaseCheck, 87 _#checkGitClean, 88 _#pullThroughProxy, 89 _#failCLBuild, 90 ] 91 } 92 mark_ci_success: { 93 "runs-on": _#linuxMachine 94 if: "${{ \(_#isCLCITestBranch) }}" 95 needs: "test" 96 steps: [ 97 _#writeNetrcFile, 98 _#passCLBuild, 99 ] 100 } 101 delete_build_branch: { 102 "runs-on": _#linuxMachine 103 if: "${{ \(_#isCLCITestBranch) && always() }}" 104 needs: "test" 105 steps: [ 106 _#step & { 107 run: """ 108 \(_#tempCueckooGitDir) 109 git push https://github.com/cue-lang/cue :${GITHUB_REF#\(_#branchRefPrefix)} 110 """ 111 }, 112 ] 113 } 114 } 115 116 // _#isCLCITestBranch is an expression that evaluates to true 117 // if the job is running as a result of a CL triggered CI build 118 _#isCLCITestBranch: "startsWith(github.ref, '\(_#branchRefPrefix)ci/')" 119 120 // _#isMaster is an expression that evaluates to true if the 121 // job is running as a result of a master commit push 122 _#isMaster: "github.ref == '\(_#branchRefPrefix+_#masterBranch)'" 123 124 _#pullThroughProxy: _#step & { 125 name: "Pull this commit through the proxy on \(_#masterBranch)" 126 run: """ 127 v=$(git rev-parse HEAD) 128 cd $(mktemp -d) 129 go mod init mod.com 130 GOPROXY=https://proxy.golang.org go get -d github.com/joomcode/cue/cmd/cue@$v 131 """ 132 if: "${{ \(_#isMaster) }}" 133 } 134 135 _#startCLBuild: _#step & { 136 name: "Update Gerrit CL message with starting message" 137 run: (_#gerrit._#setCodeReview & { 138 #args: { 139 message: "Started the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" 140 } 141 }).res 142 } 143 144 _#failCLBuild: _#step & { 145 if: "${{ \(_#isCLCITestBranch) && failure() }}" 146 name: "Post any failures for this matrix entry" 147 run: (_#gerrit._#setCodeReview & { 148 #args: { 149 message: "Build failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} for more details" 150 labels: { 151 "TryBot-Result": -1 152 } 153 } 154 }).res 155 } 156 157 _#passCLBuild: _#step & { 158 name: "Update Gerrit CL message with success message" 159 run: (_#gerrit._#setCodeReview & { 160 #args: { 161 message: "Build succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" 162 labels: { 163 "TryBot-Result": 1 164 } 165 } 166 }).res 167 } 168 169 _#gerrit: { 170 // _#setCodeReview assumes that it is invoked from a job where 171 // _#isCLCITestBranch is true 172 _#setCodeReview: { 173 #args: { 174 tag: "trybot" 175 message: string 176 labels?: { 177 "TryBot-Result": int 178 } 179 } 180 res: #""" 181 \#(_#curl) -n -H "Content-Type: application/json" --request POST --data \#(strconv.Quote(encjson.Marshal(#args))) https://review.gerrithub.io/a/changes/$(basename $(dirname $GITHUB_REF))/revisions/$(basename $GITHUB_REF)/review 182 """# 183 } 184 } 185 } 186 187 repository_dispatch: _#bashWorkflow & { 188 // These constants are defined by github.com/cue-sh/tools/cmd/cueckoo 189 _#runtrybot: "runtrybot" 190 _#unity: "unity" 191 192 _#dispatchJob: _#job & { 193 _#type: string 194 "runs-on": _#linuxMachine 195 if: "${{ github.event.client_payload.type == '\(_#type)' }}" 196 } 197 198 name: "Repository Dispatch" 199 on: ["repository_dispatch"] 200 jobs: { 201 "\(_#runtrybot)": _#dispatchJob & { 202 _#type: _#runtrybot 203 steps: [ 204 _#writeNetrcFile, 205 _#step & { 206 name: "Trigger trybot" 207 run: """ 208 \(_#tempCueckooGitDir) 209 git fetch https://review.gerrithub.io/a/cue-lang/cue ${{ github.event.client_payload.payload.ref }} 210 git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD 211 git push https://github.com/cue-lang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} 212 """ 213 }, 214 ] 215 } 216 } 217 } 218 219 release: _#bashWorkflow & { 220 221 name: "Release" 222 on: push: tags: [_#releaseTagPattern] 223 jobs: goreleaser: { 224 "runs-on": _#linuxMachine 225 steps: [ 226 _#checkoutCode & { 227 with: "fetch-depth": 0 228 }, 229 _#installGo & { 230 with: "go-version": _#pinnedReleaseGo 231 }, 232 _#step & { 233 name: "Setup qemu" 234 uses: "docker/setup-qemu-action@v1" 235 }, 236 _#step & { 237 name: "Set up Docker Buildx" 238 uses: "docker/setup-buildx-action@v1" 239 }, 240 _#step & { 241 name: "Docker Login" 242 uses: "docker/login-action@v1" 243 with: { 244 registry: "docker.io" 245 username: "cueckoo" 246 password: "${{ secrets.CUECKOO_DOCKER_PAT }}" 247 } 248 }, 249 _#step & { 250 name: "Run GoReleaser" 251 env: GITHUB_TOKEN: "${{ secrets.CUECKOO_GITHUB_PAT }}" 252 uses: "goreleaser/goreleaser-action@v2" 253 with: { 254 args: "release --rm-dist" 255 version: "v1.8.2" 256 } 257 }, 258 _#step & { 259 _#arg: { 260 event_type: "Re-test post release of ${GITHUB_REF##refs/tags/}" 261 } 262 name: "Re-test cuelang.org" 263 run: #""" 264 \#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary \#(strconv.Quote(encjson.Marshal(_#arg))) https://api.github.com/repos/cue-lang/cuelang.org/dispatches 265 """# 266 }, 267 _#step & { 268 _#arg: { 269 event_type: "Check against CUE ${GITHUB_REF##refs/tags/}" 270 client_payload: { 271 type: "unity" 272 payload: { 273 versions: """ 274 "${GITHUB_REF##refs/tags/}" 275 """ 276 } 277 } 278 } 279 name: "Trigger unity build" 280 run: #""" 281 \#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary \#(strconv.Quote(encjson.Marshal(_#arg))) https://api.github.com/repos/cue-unity/unity/dispatches 282 """# 283 }, 284 ] 285 } 286 } 287 288 tip_triggers: _#bashWorkflow & { 289 290 name: "Push to tip triggers" 291 on: push: branches: [_#masterBranch] 292 jobs: push: { 293 "runs-on": _#linuxMachine 294 steps: [ 295 { 296 name: "Rebuild tip.cuelang.org" 297 run: "\(_#curl) -X POST -d {} https://api.netlify.com/build_hooks/${{ secrets.CuelangOrgTipRebuildHook }}" 298 }, 299 { 300 _#arg: { 301 event_type: "Check against ${GITHUB_SHA}" 302 client_payload: { 303 type: "unity" 304 payload: { 305 versions: """ 306 "commit:${GITHUB_SHA}" 307 """ 308 } 309 } 310 } 311 name: "Trigger unity build" 312 run: #""" 313 \#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary \#(strconv.Quote(encjson.Marshal(_#arg))) https://api.github.com/repos/cue-unity/unity/dispatches 314 """# 315 }, 316 ] 317 } 318 } 319 320 _#bashWorkflow: json.#Workflow & { 321 jobs: [string]: defaults: run: shell: "bash" 322 } 323 324 // TODO: drop when cuelang.org/issue/390 is fixed. 325 // Declare definitions for sub-schemas 326 _#job: ((json.#Workflow & {}).jobs & {x: _}).x 327 _#step: ((_#job & {steps: _}).steps & [_])[0] 328 329 // We use the oldest supported Go version for code generation. 330 // TODO(mvdan): now that we don't use qgo via go:generate, 331 // we should try to use latestStableGo for code generation, 332 // which is closer to how developers will run go generate. 333 _#codeGenGo: "1.16.x" 334 335 // Use the latest Go version for extra checks, 336 // such as running tests with the data race detector. 337 _#latestStableGo: "1.18.x" 338 339 // Use a specific latest version for release builds. 340 // Note that we don't want ".x" for the sake of reproducibility, 341 // so we instead pin a specific Go release. 342 _#pinnedReleaseGo: "1.18.1" 343 344 _#linuxMachine: "ubuntu-18.04" 345 _#macosMachine: "macos-10.15" 346 _#windowsMachine: "windows-2019" 347 348 _#testStrategy: { 349 "fail-fast": false 350 matrix: { 351 "go-version": [_#codeGenGo, "1.17.x", _#latestStableGo] 352 os: [_#linuxMachine, _#macosMachine, _#windowsMachine] 353 } 354 } 355 356 _#setGoBuildTags: _#step & { 357 _#tags: string 358 name: "Set go build tags" 359 run: """ 360 go env -w GOFLAGS=-tags=\(_#tags) 361 """ 362 } 363 364 _#installGo: _#step & { 365 name: "Install Go" 366 uses: "actions/setup-go@v3" 367 with: { 368 "go-version": *"${{ matrix.go-version }}" | string 369 } 370 } 371 372 _#checkoutCode: _#step & { 373 name: "Checkout code" 374 uses: "actions/checkout@v3" 375 } 376 377 _#cacheGoModules: _#step & { 378 name: "Cache Go modules" 379 uses: "actions/cache@v3" 380 with: { 381 path: "~/go/pkg/mod" 382 key: "${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}" 383 "restore-keys": """ 384 ${{ runner.os }}-${{ matrix.go-version }}-go- 385 """ 386 } 387 } 388 389 _#goGenerate: _#step & { 390 name: "Generate" 391 run: "go generate ./..." 392 // The Go version corresponds to the precise version specified in 393 // the matrix. Skip windows for now until we work out why re-gen is flaky 394 if: "matrix.go-version == '\(_#codeGenGo)' && matrix.os == '\(_#linuxMachine)'" 395 } 396 397 _#goTest: _#step & { 398 name: "Test" 399 run: "go test ./..." 400 } 401 402 _#goTestRace: _#step & { 403 name: "Test with -race" 404 run: "go test -race ./..." 405 } 406 407 _#goReleaseCheck: _#step & { 408 name: "gorelease check" 409 run: "go run golang.org/x/exp/cmd/gorelease" 410 } 411 412 _#checkGitClean: _#step & { 413 name: "Check that git is clean post generate and tests" 414 run: "test -z \"$(git status --porcelain)\" || (git status; git diff; false)" 415 } 416 417 _#writeNetrcFile: _#step & { 418 name: "Write netrc file for cueckoo Gerrithub" 419 run: """ 420 cat <<EOD > ~/.netrc 421 machine review.gerrithub.io 422 login cueckoo 423 password ${{ secrets.CUECKOO_GERRITHUB_PASSWORD }} 424 EOD 425 chmod 600 ~/.netrc 426 """ 427 } 428 429 _#branchRefPrefix: "refs/heads/" 430 431 _#tempCueckooGitDir: """ 432 mkdir tmpgit 433 cd tmpgit 434 git init 435 git config user.name cueckoo 436 git config user.email cueckoo@gmail.com 437 git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" 438 """ 439 440 _#curl: "curl -f -s"