github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/Makefile (about) 1 # Copyright 2017 The WPT Dashboard Project. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 # Make targets in this file are intended to be run inside the Docker container 6 # environment. 7 8 # Make targets can be run in a host environment, but that requires ensuring 9 # the correct version of tools are installed and environment variables are 10 # set appropriately. 11 12 # Prefer simply expanded variables (:=) to avoid confusion caused by recursion. 13 # All variables can be overridden in command line by `make target FOO=bar`. 14 15 SHELL := /bin/bash 16 # WPTD_PATH will have a trailing slash, e.g. /home/user/wpt.fyi/ 17 WPTD_PATH := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 18 NODE_SELENIUM_PATH := $(WPTD_PATH)webapp/node_modules/selenium-standalone/.selenium/ 19 FIREFOX_PATH := /usr/bin/firefox 20 CHROME_PATH := /usr/bin/google-chrome 21 CHROMEDRIVER_PATH=/usr/bin/chromedriver 22 USE_FRAME_BUFFER := true 23 STAGING := false 24 VERBOSE := -v 25 26 GO_FILES := $(shell find $(WPTD_PATH) -type f -name '*.go') 27 GO_TEST_FILES := $(shell find $(WPTD_PATH) -type f -name '*_test.go') 28 # Golangci version should be updated periodically. 29 # See: https://golangci-lint.run/usage/install/#other-ci 30 GOLANGCI_LINT_VERSION := v1.52.2 31 32 build: go_build 33 34 test: go_test python_test 35 36 lint: eslint go_lint golangci_lint # TODO: Replace go_lint with golangci_lint 37 38 prepush: VERBOSE := $() # Empty out the verbose flag. 39 prepush: go_build go_test lint 40 41 python_test: python3 tox 42 tox -c results-processor/ 43 44 # Contains setup necessary only for github actions. 45 github_action_go_setup: 46 # https://github.com/web-platform-tests/wpt.fyi/issues/3089 47 if [ -d "/github/workspace" ]; then \ 48 echo "Avoiding buildvcs error for Go 1.18+ by marking github workspace safe."; \ 49 git config --global --add safe.directory /github/workspace ; \ 50 else \ 51 echo "Did not detect github workspace. Skipping." ; \ 52 fi 53 # NOTE: We prune before generate, because node_modules are packr'd into the 54 # binary (and part of the build). 55 go_build: git mockgen packr2 github_action_go_setup 56 make webapp_node_modules_prod 57 go generate ./... 58 # Check all packages without producing any output. 59 go build -v ./... 60 # Build the webapp. 61 go build -v ./webapp/web 62 63 go_build_dev: 64 @ # Disable packr to always serve local node modules and dynamic components. 65 @ # There's thus no need to prune node_modules. 66 go build -v -tags skippackr ./webapp/web 67 68 go_lint: golint go_test_tag_lint 69 golint -set_exit_status ./api/... 70 golint -set_exit_status ./shared/... 71 golint -set_exit_status ./util/... 72 golint -set_exit_status ./webapp/... 73 golint -set_exit_status ./webdriver/... 74 75 # TODO: run on /shared/, /util/, /webapp/, /webdriver/ 76 golangci_lint: golangci-lint 77 golangci-lint cache clean 78 golangci-lint run ./api/... 79 80 go_test_tag_lint: 81 @ # Printing a list of test files without +build tag, asserting empty... 82 @TAGLESS=$$(grep -PL '\/\/(\s?\+build|go:build) !?(small|medium|large|cloud)' $(GO_TEST_FILES)); \ 83 if [ -n "$$TAGLESS" ]; then echo -e "Files are missing '// +build TAG' or '//go:build TAG' tags:\n$$TAGLESS" && exit 1; fi 84 85 go_test: go_small_test go_medium_test 86 87 go_small_test: go_build gcc 88 go test -tags=small $(VERBOSE) ./... 89 90 go_medium_test: go_build dev_appserver_deps gcc 91 go test -tags=medium $(VERBOSE) $(FLAGS) ./... 92 93 # Use sub-make because otherwise make would only execute the first invocation 94 # of _go_webdriver_test. Variables will be passed into sub-make implicitly. 95 go_large_test: 96 make go_firefox_test 97 make go_chrome_test 98 99 go_firefox_test: firefox geckodriver 100 make _go_webdriver_test BROWSER=firefox 101 102 go_chrome_test: chrome chromedriver 103 make _go_webdriver_test BROWSER=chrome 104 105 go_cloud_test: gcloud_login 106 gcloud config set project wptdashboard-staging; \ 107 if [[ -f "$(WPTD_PATH)client-secret.json" ]]; then \ 108 echo "Running with client-secret.json credentials instead of possible system credentials. This should happen for CI runs."; \ 109 export GOOGLE_APPLICATION_CREDENTIALS="$(WPTD_PATH)client-secret.json"; \ 110 fi ; \ 111 GOOGLE_CLOUD_PROJECT=wptdashboard-staging GAE_SERVICE=test GAE_VERSION=1 go test -tags=cloud $(VERBOSE) $(FLAGS) ./... 112 113 puppeteer_chrome_test: go_build dev_appserver_deps webdriver_node_deps 114 cd webdriver; npm test 115 116 webdriver_node_deps: 117 cd webdriver; npm install 118 119 # _go_webdriver_test is not intended to be used directly; use go_firefox_test or 120 # go_chrome_test instead. 121 _go_webdriver_test: var-BROWSER java go_build xvfb geckodriver dev_appserver_deps gcc 122 @ # This Go test manages Xvfb itself, so we don't start/stop Xvfb for it. 123 @ # The following variables are defined here because we don't know the 124 @ # path before installing geckodriver as it includes version strings. 125 GECKODRIVER_PATH="$(shell find $(NODE_SELENIUM_PATH)geckodriver/ -type f -name '*geckodriver')"; \ 126 COMMAND="go test $(VERBOSE) -timeout=15m -tags=large ./webdriver -args \ 127 -firefox_path=$(FIREFOX_PATH) \ 128 -geckodriver_path=$$GECKODRIVER_PATH \ 129 -chrome_path=$(CHROME_PATH) \ 130 -chromedriver_path=$(CHROMEDRIVER_PATH) \ 131 -frame_buffer=$(USE_FRAME_BUFFER) \ 132 -staging=$(STAGING) \ 133 -browser=$(BROWSER) $(FLAGS)"; \ 134 if [ "$$UID" == "0" ]; then sudo -u browser $$COMMAND; else $$COMMAND; fi 135 136 # NOTE: psmisc includes killall, needed by wct.sh 137 web_components_test: xvfb firefox chrome webapp_node_modules_all psmisc 138 util/wct.sh $(USE_FRAME_BUFFER) 139 140 dev_appserver_deps: gcloud-app-engine-go gcloud-cloud-datastore-emulator gcloud-beta java 141 142 # Note: If we change to downloading chrome from Chrome For Testing, modify the 143 # `chromedriver` target below to use the `known-good-versions-with-downloads.json` endpoint. 144 # More details can be found in the comment for the `chromedriver` target. 145 chrome: wget 146 if [[ -z "$$(which google-chrome)" ]]; then \ 147 ARCHIVE=google-chrome-stable_current_amd64.deb; \ 148 wget -q https://dl.google.com/linux/direct/$${ARCHIVE}; \ 149 sudo apt-get update; \ 150 sudo dpkg --install $${ARCHIVE} 2>/dev/null || true; \ 151 sudo apt-get install --fix-broken --fix-missing -qqy; \ 152 sudo dpkg --install $${ARCHIVE} 2>/dev/null; \ 153 fi 154 155 # Pull ChromeDriver from Chrome For Testing (CfT) 156 # Need to create the CHROMEDRIVER_PATH and then move the files in because the 157 # directory structure in chromedriver_linux64.zip has changed. 158 # 159 # CfT only has ChromeDriver URLs for chrome versions >=115. But assuming `chrome` 160 # target above remains pulling the latest stable, this will not be a problem. 161 # 162 # Until we also pull chrome from CfT, we should use the latest-patch-versions-per-build-with-downloads.json. 163 # When we make the switch, we can download from the known-good-versions-with-downloads.json endpoint too. 164 # More details: https://github.com/web-platform-tests/wpt.fyi/pull/3433/files#r1282787489 165 chromedriver: wget unzip chrome jq 166 if [[ ! -f "$(CHROMEDRIVER_PATH)" ]]; then \ 167 CHROME_VERSION=$$(google-chrome --version | grep -ioE "[0-9]+\.[0-9]+\.[0-9]+"); \ 168 CHROMEDRIVER_URL=$$(curl -s https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build-with-downloads.json | jq -r ".builds[\"$${CHROME_VERSION}\"].downloads.chromedriver[] | select(.platform == \"linux64\") | .url"); \ 169 TEMP_DIR=$$(mktemp -d); \ 170 wget -q -O $${TEMP_DIR}/chromedriver_linux64.zip $${CHROMEDRIVER_URL}; \ 171 unzip -j $${TEMP_DIR}/chromedriver_linux64.zip -d $${TEMP_DIR}; \ 172 sudo mv $${TEMP_DIR}/chromedriver $(CHROMEDRIVER_PATH); \ 173 sudo chmod +x $(CHROMEDRIVER_PATH); \ 174 fi 175 176 firefox: bzip2 wget 177 if [[ -z "$$(which firefox)" ]]; then \ 178 wget -O firefox.tar.bz2 -q "https://download.mozilla.org/?product=firefox-latest&os=linux64&lang=en-US"; \ 179 mkdir -p $$HOME/browsers; \ 180 tar -xjf firefox.tar.bz2 -C $$HOME/browsers; \ 181 sudo ln -s $$HOME/browsers/firefox/firefox $(FIREFOX_PATH); \ 182 fi 183 184 geckodriver: node-wct-local 185 186 golangci-lint: curl gpg 187 if [ "$$(which golangci-lint)" == "" ]; then \ 188 curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/${GOLANGCI_LINT_VERSION}/install.sh | sh -s -- -b $$(go env GOPATH)/bin; \ 189 fi 190 191 golint: git 192 if [ "$$(which golint)" == "" ]; then \ 193 go install golang.org/x/lint/golint; \ 194 fi 195 196 mockgen: git 197 if [ "$$(which mockgen)" == "" ]; then \ 198 go install github.com/golang/mock/mockgen; \ 199 fi 200 201 packr2: git 202 if [ "$$(which packr2)" == "" ]; then \ 203 go install github.com/gobuffalo/packr/v2/packr2; \ 204 fi 205 206 package_service: var-APP_PATH 207 # Trim the potential "app.staging.yaml" suffix. 208 if [[ "$(APP_PATH)" == "api/query/cache/service"* ]]; then \ 209 APP_PATH="api/query/cache/service"; \ 210 else \ 211 APP_PATH="$(APP_PATH)"; \ 212 fi ; \ 213 if [[ "$${APP_PATH}" == "api/query/cache/service" ]]; then \ 214 TMP_DIR=$$(mktemp -d); \ 215 rm -rf $(WPTD_PATH)$${APP_PATH}/wpt.fyi; \ 216 cp -r $(WPTD_PATH)* $${TMP_DIR}/; \ 217 mkdir $(WPTD_PATH)$${APP_PATH}/wpt.fyi; \ 218 cp -r $${TMP_DIR}/* $(WPTD_PATH)$${APP_PATH}/wpt.fyi/; \ 219 rm -rf $${TMP_DIR}; \ 220 fi 221 222 sys_deps: apt_update 223 make gcloud 224 make git 225 make node 226 227 apt_update: 228 sudo apt-get -qq update 229 230 bzip2: apt-get-bzip2 231 curl: apt-get-curl 232 gcc: apt-get-gcc 233 git: apt-get-git 234 jq: apt-get-jq 235 psmisc: apt-get-psmisc 236 python3: apt-get-python3.9 237 tox: apt-get-tox 238 unzip: apt-get-unzip 239 wget: apt-get-wget 240 241 java: 242 @ # java has a different apt-get package name. 243 if [[ "$$(which java)" == "" ]]; then \ 244 sudo apt-get install -qqy --no-install-suggests openjdk-8-jdk; \ 245 fi 246 247 gpg: 248 @ # gpg has a different apt-get package name. 249 if [[ "$$(which gpg)" == "" ]]; then \ 250 sudo apt-get install -qqy --no-install-suggests gnupg; \ 251 fi 252 253 inotifywait: 254 @ # inotifywait has a different apt-get package name. 255 if [[ "$$(which inotifywait)" == "" ]]; then \ 256 sudo apt-get install -qqy --no-install-suggests inotify-tools; \ 257 fi 258 259 node: curl gpg 260 if [[ "$$(which node)" == "" ]]; then \ 261 curl -sL https://deb.nodesource.com/setup_18.x | sudo -E bash -; \ 262 sudo apt-get install -qqy nodejs; \ 263 fi 264 265 gcloud: python3 curl gpg 266 if [[ "$$(which gcloud)" == "" ]]; then \ 267 curl -s https://sdk.cloud.google.com > ./install-gcloud.sh; \ 268 bash ./install-gcloud.sh --disable-prompts --install-dir=$(HOME) > /dev/null; \ 269 rm -f ./install-gcloud.sh; \ 270 gcloud components install --quiet core gsutil; \ 271 gcloud config set disable_usage_reporting false; \ 272 fi 273 274 eslint: webapp_node_modules_all 275 cd webapp; npm run lint 276 277 dev_data: FLAGS := -remote_host=staging.wpt.fyi 278 dev_data: git 279 go run $(WPTD_PATH)util/populate_dev_data.go $(FLAGS) 280 281 gcloud_login: gcloud 282 if [[ -z "$$(gcloud config list account --format "value(core.account)")" ]]; then \ 283 gcloud auth activate-service-account --key-file $(WPTD_PATH)client-secret.json; \ 284 fi 285 286 deployment_state: go_build gcloud_login package_service var-APP_PATH 287 288 deploy_staging: git apt-get-jq 289 deploy_staging: BRANCH_NAME := $$(git rev-parse --abbrev-ref HEAD) 290 deploy_staging: deployment_state var-BRANCH_NAME 291 gcloud config set project wptdashboard-staging 292 if [[ "$(BRANCH_NAME)" == "refs/heads/main" ]]; then \ 293 util/deploy.sh -q -r -p $(APP_PATH); \ 294 else \ 295 util/deploy.sh -q -b $(BRANCH_NAME) $(APP_PATH); \ 296 fi 297 rm -rf $(WPTD_PATH)api/query/cache/service/wpt.fyi 298 299 cleanup_staging_versions: gcloud_login 300 $(WPTD_PATH)/util/cleanup-versions.sh 301 302 deploy_production: deployment_state 303 gcloud config set project wptdashboard 304 util/deploy.sh -r $(APP_PATH) 305 rm -rf $(WPTD_PATH)api/query/cache/service/wpt.fyi 306 307 webapp_node_modules_all: node 308 cd webapp; npm install 309 310 webapp_node_modules_prod: webapp_node_modules_all 311 cd webapp; npm prune --production 312 313 xvfb: 314 if [[ "$(USE_FRAME_BUFFER)" == "true" && "$$(which Xvfb)" == "" ]]; then \ 315 sudo apt-get install -qqy --no-install-suggests xvfb; \ 316 fi 317 318 gcloud-%: gcloud 319 gcloud components list --only-local-state --format="value(id)" 2>/dev/null | grep -q "$*" \ 320 || gcloud components install --quiet $* 321 322 node-%: node 323 @ echo "# Installing $*..." 324 # Hack to (more quickly) detect whether a package is already installed (available in node). 325 cd webapp; node -p "require('$*/package.json').version" 2>/dev/null || npm install --no-save $* 326 327 apt-get-%: 328 if [[ "$$(which $*)" == "" ]]; then sudo apt-get install -qqy --no-install-suggests $*; fi 329 330 env-%: 331 @ if [[ "${${*}}" = "" ]]; then echo "Environment variable $* not set"; exit 1; fi 332 333 var-%: 334 @ if [[ "$($*)" = "" ]]; then echo "Make variable $* not set"; exit 1; fi