github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/internal/ci/workflows.cue (about)

     1  // Copyright 2021 The CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package ci
    16  
    17  import (
    18  	"github.com/SchemaStore/schemastore/src/schemas/json"
    19  	encjson "encoding/json"
    20  	"strconv"
    21  )
    22  
    23  workflowsDir: *"./" | string @tag(workflowsDir)
    24  
    25  _#masterBranch:      "master"
    26  _#releaseTagPattern: "v*"
    27  
    28  workflows: [...{file: string, schema: (json.#Workflow & {})}]
    29  workflows: [
    30  	{
    31  		file:   "test.yml"
    32  		schema: test
    33  	},
    34  	{
    35  		file:   "repository_dispatch.yml"
    36  		schema: repository_dispatch
    37  	},
    38  	{
    39  		file:   "release.yml"
    40  		schema: release
    41  	},
    42  	{
    43  		file:   "tip_triggers.yml"
    44  		schema: tip_triggers
    45  	},
    46  ]
    47  
    48  test: _#bashWorkflow & {
    49  
    50  	name: "Test"
    51  	on: {
    52  		push: {
    53  			branches: ["**"] // any branch (including '/' namespaced branches)
    54  			"tags-ignore": [_#releaseTagPattern]
    55  		}
    56  		pull_request: {}
    57  	}
    58  
    59  	jobs: {
    60  		start: {
    61  			"runs-on": _#linuxMachine
    62  			steps: [...(_ & {if: "${{ \(_#isCLCITestBranch) }}"})]
    63  			steps: [
    64  				_#writeNetrcFile,
    65  				_#startCLBuild,
    66  			]
    67  		}
    68  		test: {
    69  			needs:     "start"
    70  			strategy:  _#testStrategy
    71  			"runs-on": "${{ matrix.os }}"
    72  			steps: [
    73  				_#writeNetrcFile,
    74  				_#installGo,
    75  				_#checkoutCode,
    76  				_#cacheGoModules,
    77  				_#setGoBuildTags & {
    78  					_#tags: "long"
    79  					if:     "${{ \(_#isMaster) }}"
    80  				},
    81  				_#goGenerate,
    82  				_#goTest,
    83  				_#goTestRace & {
    84  					if: "${{ matrix.go-version == '\(_#latestStableGo)' && matrix.os == '\(_#linuxMachine)' }}"
    85  				},
    86  				_#goReleaseCheck,
    87  				_#checkGitClean,
    88  				_#pullThroughProxy,
    89  				_#failCLBuild,
    90  			]
    91  		}
    92  		mark_ci_success: {
    93  			"runs-on": _#linuxMachine
    94  			if:        "${{ \(_#isCLCITestBranch) }}"
    95  			needs:     "test"
    96  			steps: [
    97  				_#writeNetrcFile,
    98  				_#passCLBuild,
    99  			]
   100  		}
   101  		delete_build_branch: {
   102  			"runs-on": _#linuxMachine
   103  			if:        "${{ \(_#isCLCITestBranch) && always() }}"
   104  			needs:     "test"
   105  			steps: [
   106  				_#step & {
   107  					run: """
   108  						\(_#tempCueckooGitDir)
   109  						git push https://github.com/cue-lang/cue :${GITHUB_REF#\(_#branchRefPrefix)}
   110  						"""
   111  				},
   112  			]
   113  		}
   114  	}
   115  
   116  	// _#isCLCITestBranch is an expression that evaluates to true
   117  	// if the job is running as a result of a CL triggered CI build
   118  	_#isCLCITestBranch: "startsWith(github.ref, '\(_#branchRefPrefix)ci/')"
   119  
   120  	// _#isMaster is an expression that evaluates to true if the
   121  	// job is running as a result of a master commit push
   122  	_#isMaster: "github.ref == '\(_#branchRefPrefix+_#masterBranch)'"
   123  
   124  	_#pullThroughProxy: _#step & {
   125  		name: "Pull this commit through the proxy on \(_#masterBranch)"
   126  		run: """
   127  			v=$(git rev-parse HEAD)
   128  			cd $(mktemp -d)
   129  			go mod init mod.com
   130  			GOPROXY=https://proxy.golang.org go get -d github.com/joomcode/cue/cmd/cue@$v
   131  			"""
   132  		if: "${{ \(_#isMaster) }}"
   133  	}
   134  
   135  	_#startCLBuild: _#step & {
   136  		name: "Update Gerrit CL message with starting message"
   137  		run:  (_#gerrit._#setCodeReview & {
   138  			#args: {
   139  				message: "Started the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"
   140  			}
   141  		}).res
   142  	}
   143  
   144  	_#failCLBuild: _#step & {
   145  		if:   "${{ \(_#isCLCITestBranch) && failure() }}"
   146  		name: "Post any failures for this matrix entry"
   147  		run:  (_#gerrit._#setCodeReview & {
   148  			#args: {
   149  				message: "Build failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} for more details"
   150  				labels: {
   151  					"TryBot-Result": -1
   152  				}
   153  			}
   154  		}).res
   155  	}
   156  
   157  	_#passCLBuild: _#step & {
   158  		name: "Update Gerrit CL message with success message"
   159  		run:  (_#gerrit._#setCodeReview & {
   160  			#args: {
   161  				message: "Build succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"
   162  				labels: {
   163  					"TryBot-Result": 1
   164  				}
   165  			}
   166  		}).res
   167  	}
   168  
   169  	_#gerrit: {
   170  		// _#setCodeReview assumes that it is invoked from a job where
   171  		// _#isCLCITestBranch is true
   172  		_#setCodeReview: {
   173  			#args: {
   174  				tag:     "trybot"
   175  				message: string
   176  				labels?: {
   177  					"TryBot-Result": int
   178  				}
   179  			}
   180  			res: #"""
   181  			\#(_#curl) -n -H "Content-Type: application/json" --request POST --data \#(strconv.Quote(encjson.Marshal(#args))) https://review.gerrithub.io/a/changes/$(basename $(dirname $GITHUB_REF))/revisions/$(basename $GITHUB_REF)/review
   182  			"""#
   183  		}
   184  	}
   185  }
   186  
   187  repository_dispatch: _#bashWorkflow & {
   188  	// These constants are defined by github.com/cue-sh/tools/cmd/cueckoo
   189  	_#runtrybot: "runtrybot"
   190  	_#unity:     "unity"
   191  
   192  	_#dispatchJob: _#job & {
   193  		_#type:    string
   194  		"runs-on": _#linuxMachine
   195  		if:        "${{ github.event.client_payload.type == '\(_#type)' }}"
   196  	}
   197  
   198  	name: "Repository Dispatch"
   199  	on: ["repository_dispatch"]
   200  	jobs: {
   201  		"\(_#runtrybot)": _#dispatchJob & {
   202  			_#type: _#runtrybot
   203  			steps: [
   204  				_#writeNetrcFile,
   205  				_#step & {
   206  					name: "Trigger trybot"
   207  					run:  """
   208  						\(_#tempCueckooGitDir)
   209  						git fetch https://review.gerrithub.io/a/cue-lang/cue ${{ github.event.client_payload.payload.ref }}
   210  						git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD
   211  						git push https://github.com/cue-lang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }}
   212  						"""
   213  				},
   214  			]
   215  		}
   216  	}
   217  }
   218  
   219  release: _#bashWorkflow & {
   220  
   221  	name: "Release"
   222  	on: push: tags: [_#releaseTagPattern]
   223  	jobs: goreleaser: {
   224  		"runs-on": _#linuxMachine
   225  		steps: [
   226  			_#checkoutCode & {
   227  				with: "fetch-depth": 0
   228  			},
   229  			_#installGo & {
   230  				with: "go-version": _#pinnedReleaseGo
   231  			},
   232  			_#step & {
   233  				name: "Setup qemu"
   234  				uses: "docker/setup-qemu-action@v1"
   235  			},
   236  			_#step & {
   237  				name: "Set up Docker Buildx"
   238  				uses: "docker/setup-buildx-action@v1"
   239  			},
   240  			_#step & {
   241  				name: "Docker Login"
   242  				uses: "docker/login-action@v1"
   243  				with: {
   244  					registry: "docker.io"
   245  					username: "cueckoo"
   246  					password: "${{ secrets.CUECKOO_DOCKER_PAT }}"
   247  				}
   248  			},
   249  			_#step & {
   250  				name: "Run GoReleaser"
   251  				env: GITHUB_TOKEN: "${{ secrets.CUECKOO_GITHUB_PAT }}"
   252  				uses: "goreleaser/goreleaser-action@v2"
   253  				with: {
   254  					args:    "release --rm-dist"
   255  					version: "v1.8.2"
   256  				}
   257  			},
   258  			_#step & {
   259  				_#arg: {
   260  					event_type: "Re-test post release of ${GITHUB_REF##refs/tags/}"
   261  				}
   262  				name: "Re-test cuelang.org"
   263  				run:  #"""
   264  					\#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary \#(strconv.Quote(encjson.Marshal(_#arg))) https://api.github.com/repos/cue-lang/cuelang.org/dispatches
   265  					"""#
   266  			},
   267  			_#step & {
   268  				_#arg: {
   269  					event_type: "Check against CUE ${GITHUB_REF##refs/tags/}"
   270  					client_payload: {
   271  						type: "unity"
   272  						payload: {
   273  							versions: """
   274  							"${GITHUB_REF##refs/tags/}"
   275  							"""
   276  						}
   277  					}
   278  				}
   279  				name: "Trigger unity build"
   280  				run:  #"""
   281  					\#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary \#(strconv.Quote(encjson.Marshal(_#arg))) https://api.github.com/repos/cue-unity/unity/dispatches
   282  					"""#
   283  			},
   284  		]
   285  	}
   286  }
   287  
   288  tip_triggers: _#bashWorkflow & {
   289  
   290  	name: "Push to tip triggers"
   291  	on: push: branches: [_#masterBranch]
   292  	jobs: push: {
   293  		"runs-on": _#linuxMachine
   294  		steps: [
   295  			{
   296  				name: "Rebuild tip.cuelang.org"
   297  				run:  "\(_#curl) -X POST -d {} https://api.netlify.com/build_hooks/${{ secrets.CuelangOrgTipRebuildHook }}"
   298  			},
   299  			{
   300  				_#arg: {
   301  					event_type: "Check against ${GITHUB_SHA}"
   302  					client_payload: {
   303  						type: "unity"
   304  						payload: {
   305  							versions: """
   306  							"commit:${GITHUB_SHA}"
   307  							"""
   308  						}
   309  					}
   310  				}
   311  				name: "Trigger unity build"
   312  				run:  #"""
   313  					\#(_#curl) -H "Content-Type: application/json" -u cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} --request POST --data-binary \#(strconv.Quote(encjson.Marshal(_#arg))) https://api.github.com/repos/cue-unity/unity/dispatches
   314  					"""#
   315  			},
   316  		]
   317  	}
   318  }
   319  
   320  _#bashWorkflow: json.#Workflow & {
   321  	jobs: [string]: defaults: run: shell: "bash"
   322  }
   323  
   324  // TODO: drop when cuelang.org/issue/390 is fixed.
   325  // Declare definitions for sub-schemas
   326  _#job:  ((json.#Workflow & {}).jobs & {x: _}).x
   327  _#step: ((_#job & {steps:                 _}).steps & [_])[0]
   328  
   329  // We use the oldest supported Go version for code generation.
   330  // TODO(mvdan): now that we don't use qgo via go:generate,
   331  // we should try to use latestStableGo for code generation,
   332  // which is closer to how developers will run go generate.
   333  _#codeGenGo: "1.16.x"
   334  
   335  // Use the latest Go version for extra checks,
   336  // such as running tests with the data race detector.
   337  _#latestStableGo: "1.18.x"
   338  
   339  // Use a specific latest version for release builds.
   340  // Note that we don't want ".x" for the sake of reproducibility,
   341  // so we instead pin a specific Go release.
   342  _#pinnedReleaseGo: "1.18.1"
   343  
   344  _#linuxMachine:   "ubuntu-18.04"
   345  _#macosMachine:   "macos-10.15"
   346  _#windowsMachine: "windows-2019"
   347  
   348  _#testStrategy: {
   349  	"fail-fast": false
   350  	matrix: {
   351  		"go-version": [_#codeGenGo, "1.17.x", _#latestStableGo]
   352  		os: [_#linuxMachine, _#macosMachine, _#windowsMachine]
   353  	}
   354  }
   355  
   356  _#setGoBuildTags: _#step & {
   357  	_#tags: string
   358  	name:   "Set go build tags"
   359  	run:    """
   360  		go env -w GOFLAGS=-tags=\(_#tags)
   361  		"""
   362  }
   363  
   364  _#installGo: _#step & {
   365  	name: "Install Go"
   366  	uses: "actions/setup-go@v3"
   367  	with: {
   368  		"go-version": *"${{ matrix.go-version }}" | string
   369  	}
   370  }
   371  
   372  _#checkoutCode: _#step & {
   373  	name: "Checkout code"
   374  	uses: "actions/checkout@v3"
   375  }
   376  
   377  _#cacheGoModules: _#step & {
   378  	name: "Cache Go modules"
   379  	uses: "actions/cache@v3"
   380  	with: {
   381  		path: "~/go/pkg/mod"
   382  		key:  "${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}"
   383  		"restore-keys": """
   384  			${{ runner.os }}-${{ matrix.go-version }}-go-
   385  			"""
   386  	}
   387  }
   388  
   389  _#goGenerate: _#step & {
   390  	name: "Generate"
   391  	run:  "go generate ./..."
   392  	// The Go version corresponds to the precise version specified in
   393  	// the matrix. Skip windows for now until we work out why re-gen is flaky
   394  	if: "matrix.go-version == '\(_#codeGenGo)' && matrix.os == '\(_#linuxMachine)'"
   395  }
   396  
   397  _#goTest: _#step & {
   398  	name: "Test"
   399  	run:  "go test ./..."
   400  }
   401  
   402  _#goTestRace: _#step & {
   403  	name: "Test with -race"
   404  	run:  "go test -race ./..."
   405  }
   406  
   407  _#goReleaseCheck: _#step & {
   408  	name: "gorelease check"
   409  	run:  "go run golang.org/x/exp/cmd/gorelease"
   410  }
   411  
   412  _#checkGitClean: _#step & {
   413  	name: "Check that git is clean post generate and tests"
   414  	run:  "test -z \"$(git status --porcelain)\" || (git status; git diff; false)"
   415  }
   416  
   417  _#writeNetrcFile: _#step & {
   418  	name: "Write netrc file for cueckoo Gerrithub"
   419  	run: """
   420  		cat <<EOD > ~/.netrc
   421  		machine review.gerrithub.io
   422  		login cueckoo
   423  		password ${{ secrets.CUECKOO_GERRITHUB_PASSWORD }}
   424  		EOD
   425  		chmod 600 ~/.netrc
   426  		"""
   427  }
   428  
   429  _#branchRefPrefix: "refs/heads/"
   430  
   431  _#tempCueckooGitDir: """
   432  	mkdir tmpgit
   433  	cd tmpgit
   434  	git init
   435  	git config user.name cueckoo
   436  	git config user.email cueckoo@gmail.com
   437  	git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)"
   438  	"""
   439  
   440  _#curl: "curl -f -s"