github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/.circleci/config.yml (about) 1 version: 2.1 2 3 references: 4 # environment specific references - aim to avoid conflicts 5 go-machine-image: &go_machine_image 6 circleci/classic:201808-01 7 go-machine-recent-image: &go_machine_recent_image 8 ubuntu-1604:201903-01 9 go-windows-image: &go_windows_image 10 windows-server-2019-vs2019:stable 11 12 # common references 13 common_envs: &common_envs 14 GOMAXPROCS: 1 15 NOMAD_SLOW_TEST: 1 16 GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml 17 GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json 18 # disable implicit git paging. CircleCI runs commands with in a tty 19 # making git assume it's an interactive session. 20 PAGER: cat 21 GIT_PAGER: cat 22 23 commands: 24 install-buf: 25 steps: 26 - run: 27 name: install buf 28 command: | 29 sudo ./scripts/vagrant-linux-priv-buf.sh 30 install-vault: 31 parameters: 32 version: 33 type: string 34 default: 1.2.4 35 steps: 36 - run: 37 name: Install Vault 38 command: | 39 set -x 40 41 case "${OSTYPE}" in 42 linux*) os="linux" ;; 43 darwin*) os="darwin" ;; 44 msys*) os="windows" ;; 45 *) echo "unknown os: ${OSTYPE}"; exit 1 ;; 46 esac 47 48 curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"<< parameters.version >>"/vault_"<< parameters.version >>"_"${os}"_amd64.zip 49 50 if [[ "${os}" != "windows" ]]; then 51 sudo unzip -d /usr/local/bin /tmp/vault.zip 52 rm -rf /tmp/vault* 53 else 54 rm -rf c:\Windows\vault.exe 55 unzip /tmp/vault.zip -d "/c/Windows" 56 rm -rf /tmp/vault* 57 fi 58 install-consul: 59 parameters: 60 version: 61 type: string 62 default: 1.8.3 63 steps: 64 - run: 65 name: Install Consul << parameters.version >> 66 command: | 67 curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip 68 sudo unzip -d /usr/local/bin /tmp/consul.zip 69 rm -rf /tmp/consul* 70 install-golang: 71 parameters: 72 target_directory: 73 type: string 74 default: /usr/local 75 steps: 76 - run: 77 name: Install golang 78 command: | 79 set -x 80 81 echo installing golang ${GOLANG_VERSION} 82 83 case "${OSTYPE}" in 84 linux*) os="linux" ;; 85 darwin*) os="darwin" ;; 86 msys*) os="windows" ;; 87 *) echo "unknown os: ${OSTYPE}"; exit 1 ;; 88 esac 89 90 if [[ "${os}" != "windows" ]] 91 then 92 sudo rm -rf << parameters.target_directory >>/go 93 sudo mkdir -p << parameters.target_directory >> 94 curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz 95 sudo tar -C << parameters.target_directory >> -xzf /tmp/golang.tar.gz 96 rm -rf /tmp/golang.tar.gz 97 else 98 rm -rf << parameters.target_directory >>/go 99 mkdir -p <<parameters.target_directory >> 100 curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip 101 unzip -q -o /tmp/go.zip -d << parameters.target_directory >> 102 rm -rf /tmp/go.zip 103 fi 104 105 install-circleci-local-cli: 106 parameters: 107 version: 108 type: string 109 default: 0.1.5879 110 111 sha256: 112 type: string 113 default: f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6 114 steps: 115 - run: 116 name: Install CircleCI CLI << parameters.version >> 117 command: | 118 CCI_VERSION="<< parameters.version >>" 119 CCI_SHA256="<< parameters.sha256 >>" 120 121 URL="https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz" 122 123 mkdir -p /tmp/circleci-cli/ 124 curl --fail --show-error --location \ 125 -o /tmp/circleci-cli/cli.tar.gz "${URL}" 126 127 echo "$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz" | sha256sum -c 128 129 tar -xz --strip-components=1 \ 130 -C /tmp/circleci-cli \ 131 -f /tmp/circleci-cli/cli.tar.gz \ 132 "circleci-cli_${CCI_VERSION}_linux_amd64/circleci" 133 134 sudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli 135 136 circleci-local-cli version 137 run-tests: 138 steps: 139 - run: 140 name: Running Nomad Tests 141 no_output_timeout: 20m 142 command: | 143 if [ -z $GOTEST_PKGS_EXCLUDE ]; 144 then 145 unset GOTEST_PKGS_EXCLUDE 146 else 147 unset GOTEST_PKGS 148 fi 149 150 if [ ! -z $GOTESTARCH ]; then 151 export GOARCH="$GOTESTARCH"; 152 fi 153 154 mkdir -p /tmp/test-reports 155 sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs 156 157 if [ ! -z $GOTEST_MOD ]; then 158 sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module 159 else 160 sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad 161 fi 162 163 jobs: 164 website-docker-image: 165 docker: 166 - image: docker.mirror.hashicorp.services/circleci/buildpack-deps 167 shell: /usr/bin/env bash -euo pipefail -c 168 steps: 169 - checkout 170 - run: 171 name: Skip building if nothing changed 172 command: | 173 # There is an edge case that would cause an issue here - if dependencies are updated to an exact copy 174 # of a previous version, for example if packge-lock.json is reverted, we need to manually push the new 175 # image to the "latest" tag 176 # Ignore job if running an enterprise build 177 IMAGE_TAG=$(cat website/Dockerfile website/package-lock.json | sha256sum | awk '{print $1;}') 178 echo "Using $IMAGE_TAG" 179 180 if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then 181 echo "Not Nomad OSS Repo, not building website docker image" 182 circleci-agent step halt 183 elif curl https://hub.docker.com/v2/repositories/hashicorp/nomad-website/tags/$IMAGE_TAG -fsL > /dev/null; then 184 echo "Dependencies have not changed, not building a new website docker image." 185 circleci-agent step halt 186 fi 187 188 - setup_remote_docker 189 - run: 190 name: Build Docker Image 191 command: | 192 IMAGE_TAG=$(cat website/Dockerfile website/package-lock.json | sha256sum | awk '{print $1;}') 193 echo "Using $IMAGE_TAG" 194 195 cd website/ 196 docker build -t hashicorp/nomad-website:$IMAGE_TAG . 197 docker tag hashicorp/nomad-website:$IMAGE_TAG hashicorp/nomad-website:latest 198 docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS 199 docker push hashicorp/nomad-website 200 201 test-windows: 202 executor: go-windows 203 204 steps: 205 - run: git config --global core.autocrlf false 206 - checkout 207 - run: 208 name: Setup 209 command: | 210 mkdir -p $GOBIN 211 mkdir -p $GOTESTSUM_PATH 212 - install-golang: 213 target_directory: "c:" 214 - run: go version 215 - install-vault: 216 version: $VAULT_VERSION 217 - run: vault version 218 - run: choco install make 219 - run: 220 name: Install golang dependencies 221 command: | 222 export PATH=$PATH:/c/go/bin:/c/gopath/bin 223 make deps 224 - run: 225 name: Pre-download docker test image 226 command: docker pull docker.mirror.hashicorp.services/hashicorpnomad/busybox-windows:server2016-0.1 227 - run: 228 name: Build nomad 229 command: | 230 export PATH=$PATH:/c/go/bin:/c/gopath/bin 231 go build -o $GOBIN\nomad.exe 232 - run: 233 name: Run tests with gotestsum 234 command: | 235 # Only test docker driver tests for now 236 export PATH=$PATH:/c/go/bin:/c/gopath/bin 237 gotestsum --format=short-verbose \ 238 --junitfile $GOTESTSUM_PATH/results.xml \ 239 github.com/hashicorp/nomad/drivers/docker \ 240 github.com/hashicorp/nomad/client/lib/fifo \ 241 github.com/hashicorp/nomad/client/logmon 242 243 - store_test_results: 244 path: c:\tmp\test-reports 245 - store_artifacts: 246 path: c:\tmp\test-reports 247 test-ui: 248 docker: 249 - image: docker.mirror.hashicorp.services/circleci/node:12-browsers 250 environment: 251 # See https://git.io/vdao3 for details. 252 JOBS: 2 253 steps: 254 - checkout 255 - restore_cache: 256 keys: 257 - v3-deps-{{ checksum "ui/yarn.lock" }} 258 - run: 259 name: yarn install 260 command: cd ui && yarn install --frozen-lockfile 261 - save_cache: 262 key: v3-deps-{{ checksum "ui/yarn.lock" }} 263 paths: 264 - ./ui/node_modules 265 - run: 266 name: lint:js 267 command: cd ui && yarn run lint:js 268 - run: 269 name: lint:hbs 270 command: cd ui && yarn run lint:hbs 271 - run: 272 name: Ember tests 273 command: mkdir -p /tmp/test-reports && cd ui && yarn test 274 - store_test_results: 275 path: /tmp/test-reports 276 - store_artifacts: 277 path: /tmp/test-reports 278 test-machine: 279 executor: "<< parameters.executor >>" 280 parameters: 281 test_packages: 282 type: string 283 default: "" 284 test_module: 285 type: string 286 default: "" 287 exclude_packages: 288 type: string 289 default: "" 290 executor: 291 type: string 292 default: "go-machine-recent" 293 goarch: 294 type: string 295 default: "amd64" 296 environment: 297 GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" 298 GOTEST_PKGS: "<< parameters.test_packages >>" 299 GOTEST_MOD: "<< parameters.test_module >>" 300 GOTESTARCH: "<< parameters.goarch >>" 301 steps: 302 - checkout 303 - install-golang 304 - install-buf 305 - install-consul 306 - install-vault 307 - run: 308 name: Install 32bit gcc libs 309 command: | 310 if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then 311 sudo apt-get update 312 sudo apt-get install -y gcc-multilib 313 else 314 echo "Skipping 32bit lib installation while building for not 386" 315 fi 316 - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap 317 - run-tests 318 - store_test_results: 319 path: /tmp/test-reports 320 - store_artifacts: 321 path: /tmp/test-reports 322 test-e2e: 323 executor: go 324 steps: 325 - checkout 326 - run: apt-get update; apt-get install -y sudo unzip 327 # e2e tests require privileged mount/umount permissions when running as root 328 # TODO: switch to using machine executor and run as root to test e2e path 329 - run: 330 name: prepare non-root user 331 command: | 332 groupadd --gid 3434 circleci 333 useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci 334 echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci 335 echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep 336 chown -R circleci:circleci /go 337 338 - run: sudo -E -H -u circleci PATH=${PATH} make deps 339 - run: sudo -E -H -u circleci PATH=${PATH} make integration-test 340 - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test 341 test-container: 342 executor: go 343 parameters: 344 test_packages: 345 type: string 346 default: "" 347 exclude_packages: 348 type: string 349 default: "" 350 goarch: 351 type: string 352 default: "amd64" 353 environment: 354 GOTEST_PKGS: "<< parameters.test_packages >>" 355 GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" 356 GOTESTARCH: "<< parameters.goarch >>" 357 steps: 358 - checkout 359 - run: apt-get update; apt-get install -y shellcheck sudo unzip 360 - run: make deps 361 - install-buf 362 - install-consul 363 - install-vault 364 - run-tests 365 - store_test_results: 366 path: /tmp/test-reports 367 - store_artifacts: 368 path: /tmp/test-reports 369 lint-go: 370 executor: go 371 steps: 372 - checkout 373 - run: apt-get update; apt-get install -y shellcheck sudo unzip 374 - install-buf 375 - install-circleci-local-cli 376 - run: make deps lint-deps 377 - run: make check 378 - run: make checkscripts 379 - run: mkdir -p ui/dist && make generate-all static-assets 380 build-darwin-binaries: 381 executor: go-macos 382 steps: 383 - checkout 384 - run: 385 name: configure PATH 386 command: 387 echo 'export PATH="${GOPATH}/bin:${HOME}/goinstall/go/bin:$PATH"' >> ${BASH_ENV} 388 389 - install-golang: 390 target_directory: ~/goinstall 391 - run: source ${BASH_ENV} && make deps 392 - run: brew install protobuf 393 - run: PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs 394 395 - run: 396 name: prepare ui cache 397 command: git log -n 1 --pretty=format:%H ui > /tmp/ui-sha 398 399 - restore_cache: 400 name: restore compiled ui assets 401 keys: 402 - v1-binary-ui-assets-darwin-{{ checksum "/tmp/ui-sha" }} 403 - run: 404 name: prepare ui 405 command: | 406 if [[ -f ~/caches/ui-assets/bindata_assetfs.go ]]; then 407 cp ~/caches/ui-assets/bindata_assetfs.go ./command/agent/bindata_assetfs.go 408 exit 0 409 fi 410 411 ./scripts/vagrant-linux-unpriv-ui.sh 412 413 export PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" 414 source ${BASH_ENV} 415 . ~/.nvm/nvm.sh 416 cd ui && yarn install --frozen-lockfile && cd .. 417 JOBS=2 make ember-dist static-assets 418 419 mkdir -p ~/caches/ui-assets 420 cp ./command/agent/bindata_assetfs.go ~/caches/ui-assets/bindata_assetfs.go 421 - save_cache: 422 name: save compiled ui assets 423 key: v1-binary-ui-assets-darwin-{{ checksum "/tmp/ui-sha" }} 424 paths: 425 - ~/caches/ui-assets 426 - run: 427 name: build binaries 428 command: | 429 source ${BASH_ENV} 430 export GO_TAGS="ui $(grep -e '^GO_TAGS ?=' ./GNUmakefile | cut -d= -f2)" 431 make pkg/darwin_amd64.zip 432 - store_artifacts: 433 path: pkg/darwin_amd64.zip 434 destination: /builds/nomad_darwin_amd64.zip 435 build-binaries: 436 executor: go 437 steps: 438 - checkout 439 - run: apt-get update; apt-get install -y sudo unzip zip 440 - run: make deps 441 - install-buf 442 - run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs 443 444 - run: 445 name: prepare ui cache 446 command: git log -n 1 --pretty=format:%H ui > /tmp/ui-sha 447 448 - restore_cache: 449 name: restore compiled ui assets 450 keys: 451 - v1-binary-ui-assets-linux-{{ checksum "/tmp/ui-sha" }} 452 - run: 453 name: prepare ui 454 command: | 455 if [[ -f /tmp/ui-assets/bindata_assetfs.go ]]; then 456 cp /tmp/ui-assets/bindata_assetfs.go ./command/agent/bindata_assetfs.go 457 exit 0 458 fi 459 460 ./scripts/vagrant-linux-unpriv-ui.sh 461 462 export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH" 463 export PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" 464 . ~/.nvm/nvm.sh 465 cd ui && yarn install --frozen-lockfile && cd .. 466 JOBS=2 make ember-dist static-assets 467 468 mkdir -p /tmp/ui-assets 469 cp ./command/agent/bindata_assetfs.go /tmp/ui-assets/bindata_assetfs.go 470 471 - save_cache: 472 name: save compiled ui assets 473 key: v1-binary-ui-assets-linux-{{ checksum "/tmp/ui-sha" }} 474 paths: 475 - /tmp/ui-assets 476 - run: 477 name: build binaries 478 command: | 479 export GO_TAGS="ui $(grep -e '^GO_TAGS ?=' ./GNUmakefile | cut -d= -f2)" 480 make pkg/windows_amd64.zip pkg/linux_amd64.zip 481 - store_artifacts: 482 path: pkg 483 destination: /builds 484 - store_artifacts: 485 path: /tmp/ui-assets 486 destination: /ui-assets 487 algolia_index: 488 docker: 489 - image: docker.mirror.hashicorp.services/node:12 490 steps: 491 - checkout 492 - run: 493 name: Push content to Algolia Index 494 command: | 495 if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then 496 echo "Not Nomad OSS Repo, not indexing Algolia" 497 exit 0 498 fi 499 500 cd website/ 501 npm install 502 node scripts/index_search_content.js 503 executors: 504 go: 505 working_directory: /go/src/github.com/hashicorp/nomad 506 docker: 507 - image: docker.mirror.hashicorp.services/golang:1.15.6 508 environment: 509 <<: *common_envs 510 GOPATH: /go 511 512 go-machine: 513 working_directory: ~/go/src/github.com/hashicorp/nomad 514 machine: 515 image: *go_machine_image 516 environment: &machine_env 517 <<: *common_envs 518 GOPATH: /home/circleci/go 519 GOLANG_VERSION: 1.15.6 520 521 # uses a more recent image with unattended upgrades disabled properly 522 # but seems to break docker builds 523 go-machine-recent: 524 working_directory: ~/go/src/github.com/hashicorp/nomad 525 machine: 526 image: *go_machine_recent_image 527 environment: *machine_env 528 529 go-macos: 530 working_directory: ~/go/src/github.com/hashicorp/nomad 531 macos: 532 xcode: 11.3.1 533 environment: 534 <<: *common_envs 535 GOPATH: /Users/distiller/go 536 GOLANG_VERSION: 1.15.6 537 538 go-windows: 539 machine: 540 image: *go_windows_image 541 resource_class: windows.medium 542 shell: bash --login -eo pipefail 543 working_directory: c:\gopath\src\github.com\hashicorp\nomad 544 environment: 545 GOPATH: c:\gopath 546 GOBIN: c:\gopath\bin 547 GOTESTSUM_PATH: c:\tmp\test-reports 548 GOLANG_VERSION: 1.15.6 549 GOTESTSUM_VERSION: 0.4.2 550 VAULT_VERSION: 1.4.1 551 552 workflows: 553 build-test: 554 jobs: 555 - build-binaries: 556 # almost always build binaries as they may be needed 557 # for e2e tests 558 filters: 559 branches: 560 ignore: 561 - stable-website 562 - /^.-ui\b.*/ 563 564 - lint-go: 565 # check branches are almost all the backend branches 566 filters: &backend_check_branches_filter 567 branches: 568 ignore: 569 - /^.-ui\b.*/ 570 - /^docs-.*/ 571 - stable-website 572 573 # Note: comment-out this job in ENT 574 - build-darwin-binaries: 575 filters: *backend_check_branches_filter 576 577 - test-e2e: 578 filters: *backend_check_branches_filter 579 580 - test-ui: 581 filters: 582 branches: 583 ignore: 584 - stable-website 585 - /^docs-.*/ 586 - /^e2e-.*/ 587 588 # Note: comment-out this job in ENT 589 - test-windows: 590 filters: 591 branches: 592 ignore: 593 - /^.-ui\b.*/ 594 - /^docs-.*/ 595 - /^e2e-.*/ 596 - stable-website 597 598 - test-machine: 599 name: "test-client" 600 test_packages: "./client/..." 601 # test branches are the branches that can impact unit tests 602 filters: &backend_test_branches_filter 603 branches: 604 ignore: 605 - /^.-ui\b.*/ 606 - /^docs-.*/ 607 - /^e2e-.*/ 608 - stable-website 609 - test-machine: 610 name: "test-nomad" 611 test_packages: "./nomad/..." 612 filters: *backend_test_branches_filter 613 - test-machine: 614 # API Tests run in a VM rather than container due to the FS tests 615 # requiring `mount` priviliges. 616 name: "test-api" 617 test_module: "api" 618 filters: *backend_test_branches_filter 619 - test-container: 620 name: "test-devices" 621 test_packages: "./devices/..." 622 filters: *backend_test_branches_filter 623 - test-machine: 624 name: "test-other" 625 exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/shared/executor|./nomad|./devices|./e2e" 626 filters: *backend_test_branches_filter 627 - test-machine: 628 name: "test-docker" 629 test_packages: "./drivers/docker" 630 # docker is misbehaving in docker-machine-recent image 631 # and we get unexpected failures 632 # e.g. https://circleci.com/gh/hashicorp/nomad/3854 633 executor: go-machine 634 filters: *backend_test_branches_filter 635 - test-machine: 636 name: "test-exec" 637 test_packages: "./drivers/exec" 638 filters: *backend_test_branches_filter 639 - test-machine: 640 name: "test-shared-exec" 641 test_packages: "./drivers/shared/executor" 642 filters: *backend_test_branches_filter 643 - test-machine: 644 name: "test-32bit" 645 # Currently we only explicitly test fingerprinting on 32bit 646 # architectures. 647 test_packages: "./client/fingerprint" 648 goarch: "386" 649 filters: *backend_test_branches_filter 650 website: 651 when: 652 equal: [ "https://github.com/hashicorp/nomad", << pipeline.project.git_url >> ] 653 jobs: 654 - website-docker-image: 655 context: static-sites 656 filters: 657 branches: 658 only: 659 - master 660 661 - algolia_index: 662 context: static-sites 663 filters: 664 branches: 665 only: 666 - stable-website