github.com/mirantis/virtlet@v1.5.2-0.20191204181327-1659b8a48e9b/.circleci/config.yml (about) 1 version: 2.1 2 defaults: &defaults 3 docker: 4 - image: ubuntu:xenial 5 working_directory: ~/virtlet 6 environment: 7 DOCKER_VERSION: "17.03.0-ce" 8 KUBECTL_VERSION: "v1.9.3" 9 KUBECTL_SHA1: "a27d808eb011dbeea876fe5326349ed167a7ed28" 10 # remove DIND_CRI to use dockershim 11 DIND_CRI: containerd 12 # Uncomment the following to use ginkgo.focus for e2e 13 # E2E_FOCUS: "Specify a regexp.*here" 14 # The branch which is used for publishing docs 15 DOCS_SOURCE_BRANCH: master 16 17 prereqs: &prereqs 18 name: Install prerequisites 19 command: | 20 apt-get -qq update 21 apt-get install -y curl ca-certificates openssh-client rsync socat wget git 22 23 setup_env: &setup_env 24 name: Set up the environment 25 command: | 26 # Install Docker client 27 curl -sSL -o "/tmp/docker-${DOCKER_VERSION}.tgz" "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" 28 tar -xz -C /tmp -f "/tmp/docker-${DOCKER_VERSION}.tgz" 29 mv /tmp/docker/* /usr/bin 30 31 # Disable apparmor 32 docker run --rm --pid=host -it --privileged ubuntu:xenial \ 33 nsenter -t 1 -m -u -i -n /bin/bash -c \ 34 "service apparmor stop && 35 service apparmor teardown && 36 update-rc.d -f apparmor remove && 37 apt-get remove -y apparmor" 38 39 # Start port forwarder 40 build/portforward.sh start 41 42 restore_image_cache: &restore_image_cache 43 name: Restore image cache 44 keys: 45 # bump '-vNNN' to update the cache (also everywhere this key is used in the file) 46 - images-v1-{{ checksum "images/Dockerfile.build-base" }}-{{ checksum "images/Dockerfile.virtlet-base" }} 47 48 restore_vendor_cache: &restore_vendor_cache 49 name: Restore vendor cache 50 keys: 51 # bump '-vNNN' to update the cache (also everywhere this key is used in the file) 52 - vendor-v1-{{ checksum "glide.yaml" }}-{{ checksum "glide.lock" }} 53 54 restore_images_and_vendor: &restore_images_and_vendor 55 name: Prepare build image & vendor dir 56 command: | 57 if [[ -f _output/images.tar ]]; then 58 docker load -i _output/images.tar 59 fi 60 if [[ -f _output/vendor.tar ]]; then 61 build/portforward.sh 18730& 62 build/cmd.sh start-build-container 63 docker exec -i virtlet-build tar -C /go/src/github.com/Mirantis/virtlet -xv <_output/vendor.tar 64 fi 65 66 extract_binaries_from_the_image: &extract_binaries_from_the_image 67 name: Extract binaries from the image (in non-workflow case) 68 command: | 69 if [[ -f _output/vmwrapper ]]; then 70 exit 0 71 fi 72 tag="$(echo "${CIRCLE_TAG:-${CIRCLE_BRANCH}}"|tr / _)" 73 image="mirantis/virtlet:${tag}" 74 echo "Extracting binaries from ${image}" 75 mkdir -p _output/ 76 docker run "${image}" tar -C / -c vmwrapper virtlet-e2e-tests virtletctl | tar -C _output/ -xv 77 78 push_images: &push_images 79 <<: *defaults 80 steps: 81 - run: 82 <<: *prereqs 83 # the checkout is needed for getting cache key from dockerfiles 84 - checkout 85 - setup_remote_docker 86 - run: 87 name: Install Docker client 88 command: | 89 apt-get -qq update 90 apt-get install -y curl ca-certificates 91 curl -sSL -o "/tmp/docker-${DOCKER_VERSION}.tgz" "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" 92 tar -xz -C /tmp -f "/tmp/docker-${DOCKER_VERSION}.tgz" 93 mv /tmp/docker/* /usr/bin 94 - restore_cache: 95 <<: *restore_image_cache 96 - attach_workspace: 97 at: _output 98 - run: 99 name: Restore images 100 command: | 101 if [[ ! ${DOCKER_USER:-} || ! ${DOCKER_PASS} ]]; then 102 echo >&2 "*** WARNING: can't push images for outside forks" 103 exit 0 104 fi 105 docker load -i _output/virtlet.tar 106 if [[ -f _output/images.tar ]]; then 107 docker load -i _output/images.tar 108 fi 109 - run: 110 name: Retag the image 111 command: | 112 if [[ ! ${DOCKER_USER:-} || ! ${DOCKER_PASS} ]]; then 113 echo >&2 "*** WARNING: can't push images for outside forks" 114 exit 0 115 fi 116 tag="$(echo "${CIRCLE_TAG:-${CIRCLE_BRANCH}}"|tr / _)" 117 # we want to push 'latest' image for the master branch 118 if [[ ${tag} != master ]]; then 119 docker tag mirantis/virtlet:latest "mirantis/virtlet:${tag}" 120 docker rmi mirantis/virtlet:latest 121 fi 122 - run: 123 name: Push the images 124 command: | 125 if [[ ! ${DOCKER_USER:-} || ! ${DOCKER_PASS} ]]; then 126 echo >&2 "*** WARNING: can't push images for outside forks" 127 exit 0 128 fi 129 docker login -u $DOCKER_USER -p $DOCKER_PASS 130 for img in build-base virtlet-base virtlet; do 131 docker images -q -f label=virtlet.image="${img}" --format '{{.Repository}}:{{.Tag}}' 132 done | grep '^mirantis/virtlet' | while read img; do 133 echo >&2 "*** Pushing: ${img}" 134 docker push "${img}" 135 done 136 137 e2e: &e2e 138 <<: *defaults 139 steps: 140 - run: 141 <<: *prereqs 142 - checkout 143 - setup_remote_docker 144 - run: 145 <<: *setup_env 146 - attach_workspace: 147 at: _output 148 - run: 149 <<: *extract_binaries_from_the_image 150 - run: 151 name: Restore virtlet image 152 command: | 153 if [[ -f _output/virtlet.tar ]]; then 154 docker load -i _output/virtlet.tar 155 fi 156 - run: 157 name: Start the demo 158 command: | 159 build/portforward.sh 8080& 160 if [[ ${CIRCLE_JOB} = e2e_calico ]]; then 161 export CNI_PLUGIN=calico 162 echo >&2 "*** Using Calico CNI" 163 elif [[ ${CIRCLE_JOB} = e2e_flannel ]]; then 164 export CNI_PLUGIN=flannel 165 echo >&2 "*** Using Flannel CNI" 166 elif [[ ${CIRCLE_JOB} = e2e_weave ]]; then 167 export CNI_PLUGIN=weave 168 echo >&2 "*** Using Weave CNI" 169 elif [[ ${CIRCLE_JOB} = e2e_multi_cni ]]; then 170 export MULTI_CNI=1 171 echo >&2 "*** Using multiple CNIs (flannel + calico)" 172 elif [[ ${CIRCLE_JOB} = e2e_1_13 ]]; then 173 export KUBE_VERSION=1.13 174 fi 175 # APISERVER_PORT is set explicitly to avoid dynamic allocation 176 # of the port by kdc 177 APISERVER_PORT=8080 \ 178 SKIP_SNAPSHOT=1 \ 179 NONINTERACTIVE=1 \ 180 NO_VM_CONSOLE=1 \ 181 INJECT_LOCAL_IMAGE=1 \ 182 VIRTLET_DEMO_BRANCH=master \ 183 ENABLE_CEPH=1 \ 184 DEMO_LOG_LEVEL=6 \ 185 FEATURE_GATES="BlockVolume=true" \ 186 KUBELET_FEATURE_GATES="BlockVolume=true" \ 187 BASE_LOCATION="$PWD" \ 188 deploy/demo.sh 189 - run: 190 name: Install kubectl 191 command: | 192 curl -L https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl 193 echo ${KUBECTL_SHA1} /usr/local/bin/kubectl |sha1sum -c 194 chmod +x /usr/local/bin/kubectl 195 196 - run: 197 name: Prepare all of the nodes for Virtlet pod 198 command: | 199 build/portforward.sh 8080& 200 VIRTLET_SKIP_RSYNC=1 build/cmd.sh prepare-all-nodes 201 202 - run: 203 name: Run e2e tests 204 command: | 205 build/portforward.sh 8080& 206 mkdir -p ~/junit 207 e2e_opts=("-ginkgo.skip=\[Heavy\]|\[MultiCNI\]|\[Disruptive\]|\[Flaky\]") 208 if [[ ${CIRCLE_JOB} = e2e_multi_cni ]]; then 209 # per-node config test requires an additional worker node 210 e2e_opts="-ginkgo.skip=\[Heavy\]|\[Disruptive\]|\[Flaky\]|Per-node configuration" 211 fi 212 if [[ ${E2E_FOCUS:-} ]]; then 213 e2e_opts+=("-ginkgo.focus=${E2E_FOCUS}") 214 fi 215 if [[ ${CIRCLE_JOB} = e2e_debian ]]; then 216 e2e_opts+=(-image cdimage.debian.org/cdimage/openstack/archive/9.8.0/debian-9.8.0-openstack-amd64.qcow2 217 -sshuser debian 218 -memoryLimit 256 219 -use-dhcp-network-config) 220 fi 221 _output/virtlet-e2e-tests -test.v "${e2e_opts[@]}" \ 222 -junitOutput ~/junit/junit.xml \ 223 -include-unsafe-tests=true 224 225 - store_test_results: 226 path: ~/junit 227 228 - run: 229 name: Dump the cluster state and display Virtlet version 230 when: always 231 command: | 232 build/portforward.sh 8080& 233 mkdir -p /tmp/cluster_state 234 bash -x ./dind-cluster*.sh dump | gzip >/tmp/cluster_state/kdc-dump.gz 235 _output/virtletctl diag dump --json | gzip >/tmp/cluster_state/virtlet-dump.json.gz 236 _output/virtletctl version 237 238 - store_artifacts: 239 path: /tmp/cluster_state 240 241 jobs: 242 prepare_build: 243 <<: *defaults 244 steps: 245 - run: 246 <<: *prereqs 247 - checkout 248 - setup_remote_docker 249 - run: 250 <<: *setup_env 251 - restore_cache: 252 <<: *restore_image_cache 253 - restore_cache: 254 <<: *restore_vendor_cache 255 - run: 256 name: Prepare images and the vendor dir 257 command: | 258 if [[ ! -f _output/images.tar || ! -f _output/vendor.tar ]]; then 259 build/portforward.sh 18730& 260 build/cmd.sh prepare-vendor 261 fi 262 - run: 263 name: Store base images 264 command: | 265 if [[ ! -f _output/images.tar ]]; then 266 mkdir -p _output 267 build_base_image="$(docker images -q -f label=virtlet.image=build-base --format '{{.Repository}}:{{.Tag}}')" 268 virtlet_base_image="$(docker images -q -f label=virtlet.image=virtlet-base --format '{{.Repository}}:{{.Tag}}')" 269 docker save "${build_base_image}" "${virtlet_base_image}" >_output/images.tar 270 fi 271 - save_cache: 272 name: Save image cache 273 key: images-v1-{{ checksum "images/Dockerfile.build-base" }}-{{ checksum "images/Dockerfile.virtlet-base" }} 274 paths: 275 - _output/images.tar 276 - run: 277 name: Store vendor archive 278 command: | 279 if [[ ! -f _output/vendor.tar ]]; then 280 mkdir -p _output 281 docker exec virtlet-build tar -C /go/src/github.com/Mirantis/virtlet -c vendor >_output/vendor.tar 282 fi 283 - save_cache: 284 name: Save vendor cache 285 key: vendor-v1-{{ checksum "glide.yaml" }}-{{ checksum "glide.lock" }} 286 paths: 287 - _output/vendor.tar 288 289 build: 290 <<: *defaults 291 steps: 292 - run: 293 <<: *prereqs 294 - checkout 295 - setup_remote_docker 296 - run: 297 <<: *setup_env 298 - restore_cache: 299 <<: *restore_image_cache 300 - restore_cache: 301 <<: *restore_vendor_cache 302 - run: 303 <<: *restore_images_and_vendor 304 305 - run: 306 name: Verify that the generated code for CRDs is up to date 307 command: | 308 build/portforward.sh 18730& 309 build/cmd.sh update-generated 310 if [[ $(git status --porcelain) ]]; then 311 git status 312 echo >&2 "Generated files out of sync, please run build/cmd.sh update-generated" 313 exit 1 314 fi 315 316 - run: 317 name: Build virtlet 318 command: | 319 build/portforward.sh 18730& 320 SET_VIRTLET_IMAGE_TAG="${CIRCLE_TAG:-}" build/cmd.sh build 321 build/cmd.sh copy 322 - run: 323 name: Save image 324 command: | 325 docker save mirantis/virtlet >_output/virtlet.tar 326 - persist_to_workspace: 327 root: _output 328 paths: 329 - virtlet.tar 330 - virtlet-e2e-tests 331 - virtletctl 332 - virtletctl.darwin 333 # needed by integration tests 334 - vmwrapper 335 - store_artifacts: 336 path: /go/src/github.com/Mirantis/virtlet/_output 337 destination: virtlet 338 339 test: 340 <<: *defaults 341 steps: 342 - run: 343 <<: *prereqs 344 - checkout 345 - setup_remote_docker 346 - run: 347 <<: *setup_env 348 - restore_cache: 349 <<: *restore_image_cache 350 - restore_cache: 351 <<: *restore_vendor_cache 352 - run: 353 <<: *restore_images_and_vendor 354 - run: 355 name: Run tests 356 command: | 357 build/portforward.sh 18730& 358 build/cmd.sh test 359 360 integration: 361 <<: *defaults 362 steps: 363 - run: 364 <<: *prereqs 365 - checkout 366 - setup_remote_docker 367 - run: 368 <<: *setup_env 369 - restore_cache: 370 <<: *restore_image_cache 371 - restore_cache: 372 <<: *restore_vendor_cache 373 - run: 374 <<: *restore_images_and_vendor 375 - attach_workspace: 376 at: _output 377 - run: 378 <<: *extract_binaries_from_the_image 379 - run: 380 name: Run tests 381 command: | 382 build/portforward.sh 18730& 383 rm -f _output/*.tar 384 build/cmd.sh copy-back 385 build/cmd.sh integration 386 387 e2e: 388 <<: *e2e 389 390 e2e_calico: 391 <<: *e2e 392 393 e2e_flannel: 394 <<: *e2e 395 396 e2e_weave: 397 <<: *e2e 398 399 e2e_multi_cni: 400 <<: *e2e 401 402 e2e_1_13: 403 <<: *e2e 404 405 e2e_debian: 406 <<: *e2e 407 408 push_branch: 409 <<: *push_images 410 411 push_tested: 412 <<: *push_images 413 414 build_docs: 415 <<: *defaults 416 steps: 417 - run: 418 <<: *prereqs 419 - checkout 420 - setup_remote_docker 421 - run: 422 <<: *setup_env 423 - restore_cache: 424 <<: *restore_image_cache 425 - run: 426 name: Update docs and push them to docs branch 427 command: | 428 build/portforward.sh 18730& 429 git config --global user.email bot@virtlet.cloud 430 git config --global user.name virtlet-bot 431 git fetch -f https://github.com/Mirantis/virtlet.git docs:docs 432 build/cmd.sh build-docs 433 if [[ ${CIRCLE_BRANCH:-} != ${DOCS_SOURCE_BRANCH} ]]; then 434 echo >&2 "Not updating docs branch: current branch is ${CIRCLE_BRANCH:-} not ${DOCS_SOURCE_BRANCH}" 435 elif [[ ! ${GITHUB_TOKEN:-} ]]; then 436 echo >&2 "Not updating docs branch: missing GITHUB_TOKEN env var" 437 else 438 # if the docs didn't change, this push will do nothing 439 git push https://${GITHUB_TOKEN}@github.com/Mirantis/virtlet.git docs:docs 440 fi 441 tar -czf /tmp/docs.tar.gz _docs 442 - store_artifacts: 443 path: /tmp/docs.tar.gz 444 445 release: 446 <<: *defaults 447 steps: 448 - run: 449 <<: *prereqs 450 - checkout 451 - setup_remote_docker 452 - run: 453 <<: *setup_env 454 - restore_cache: 455 <<: *restore_image_cache 456 - attach_workspace: 457 at: _output 458 - run: 459 <<: *restore_images_and_vendor 460 - run: 461 name: Make a release 462 command: | 463 build/portforward.sh 18730& 464 build/cmd.sh start-build-container 465 docker cp _output virtlet-build:/go/src/github.com/Mirantis/virtlet/ 466 build/cmd.sh release "${CIRCLE_TAG}" 467 468 workflows: 469 version: 2 470 build-and-test: 471 jobs: 472 - prepare_build: 473 filters: 474 tags: 475 only: /^v[0-9].*/ 476 - build: 477 requires: 478 - prepare_build 479 filters: 480 branches: 481 ignore: /^.*-docs$/ 482 tags: 483 only: /^v[0-9].*/ 484 - test: 485 requires: 486 - prepare_build 487 filters: 488 branches: 489 ignore: /^.*-docs$/ 490 tags: 491 only: /^v[0-9].*/ 492 - integration: 493 requires: 494 - build 495 filters: 496 branches: 497 ignore: /^.*-docs$/ 498 tags: 499 only: /^v[0-9].*/ 500 - e2e: 501 requires: 502 - build 503 filters: 504 branches: 505 ignore: /^.*-docs$/ 506 tags: 507 only: /^v[0-9].*/ 508 - e2e_calico: 509 requires: 510 - build 511 filters: 512 branches: 513 ignore: /^.*-docs$/ 514 tags: 515 only: /^v[0-9].*/ 516 - e2e_flannel: 517 requires: 518 - build 519 filters: 520 branches: 521 only: /^master$|^.*-net$/ 522 tags: 523 only: /^v[0-9].*/ 524 - e2e_weave: 525 requires: 526 - build 527 filters: 528 branches: 529 only: /^master$|^.*-net$/ 530 tags: 531 only: /^v[0-9].*/ 532 # XXX: temporarily disabled, to be fixed 533 # - e2e_multi_cni: 534 # requires: 535 # - build 536 # filters: 537 # branches: 538 # only: /^master$|^.*-net$|^.*-ext-e2e$/ 539 # tags: 540 # only: /^v[0-9].*/ 541 - e2e_1_13: 542 requires: 543 - build 544 filters: 545 branches: 546 ignore: /^.*-docs$/ 547 tags: 548 only: /^v[0-9].*/ 549 - e2e_debian: 550 requires: 551 - build 552 filters: 553 branches: 554 only: /^master$|^.*-ext-e2e$/ 555 tags: 556 only: /^v[0-9].*/ 557 - push_branch: 558 requires: 559 - build 560 filters: 561 branches: 562 ignore: /^master$|^pull\/|^.*-docs$/ 563 - push_tested: 564 requires: 565 - test 566 - e2e 567 - e2e_calico 568 - e2e_flannel 569 - e2e_weave 570 # XXX: temporarily disabled, to be fixed 571 # - e2e_multi_cni 572 - e2e_1_13 573 - e2e_debian 574 - integration 575 filters: 576 branches: 577 only: master 578 tags: 579 only: /^v[0-9].*/ 580 - release: 581 requires: 582 - push_tested 583 filters: 584 branches: 585 ignore: /.*/ 586 tags: 587 only: /^v[0-9].*/ 588 - build_docs: 589 requires: 590 - prepare_build