github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/ci/templates/gpbackup-tpl.yml (about) 1 --- 2 groups: 3 - name: All 4 jobs: 5 - build_binaries 6 - build_gppkgs 7 - GPDB4.3 8 - GPDB5 9 {% if is_prod or "gpbackup-release" == pipeline_name %} 10 - GPDB5-sles11 11 - GPDB6-ubuntu 12 - ddboost_plugin_and_boostfs_tests_43 13 - ddboost_plugin_and_boostfs_tests_5x 14 {% endif %} 15 {% if is_prod and "gpbackup-release" != pipeline_name %} 16 - scale-43 17 - scale-5x 18 - scale-6x 19 - scale-GPDB7 20 - s3_plugin_perf 21 - ddboost_plugin_perf 22 {% endif %} 23 - gpbackup-manager-tests 24 {% if "gpbackup-release" != pipeline_name %} 25 - 5X-head-gpbackup-fixed-test 26 {% endif %} 27 - GPDB6 28 - GPDB6-7-seg-cluster 29 - GPDB7 30 - s3_plugin_tests 31 - backward-compatibility 32 - ddboost_plugin_and_boostfs_tests_6x 33 {% if "gpbackup-release" == pipeline_name %} 34 - push-to-github 35 - green_gate 36 - push-to-pivnet 37 {% endif %} 38 {% if "gpbackup-release" != pipeline_name %} 39 - final_gate 40 {% endif %} 41 42 - name: GPDB4.3 43 jobs: 44 - GPDB4.3 45 {% if is_prod or "gpbackup-release" == pipeline_name %} 46 - ddboost_plugin_and_boostfs_tests_43 47 {% endif %} 48 {% if is_prod and "gpbackup-release" != pipeline_name %} 49 - scale-43 50 {% endif %} 51 52 - name: GPDB5 53 jobs: 54 - GPDB5 55 {% if is_prod or "gpbackup-release" == pipeline_name %} 56 - GPDB5-sles11 57 - ddboost_plugin_and_boostfs_tests_5x 58 {% endif %} 59 {% if is_prod and "gpbackup-release" != pipeline_name %} 60 - scale-5x 61 {% endif %} 62 {% if "gpbackup-release" != pipeline_name %} 63 - 5X-head-gpbackup-fixed-test 64 {% endif %} 65 - s3_plugin_tests 66 - backward-compatibility 67 68 - name: GPDB6 69 jobs: 70 - GPDB6 71 - GPDB6-7-seg-cluster 72 {% if is_prod or "gpbackup-release" == pipeline_name %} 73 - GPDB6-ubuntu 74 {% endif %} 75 {% if is_prod and "gpbackup-release" != pipeline_name %} 76 - scale-6x 77 {% endif %} 78 - ddboost_plugin_and_boostfs_tests_6x 79 - s3_plugin_perf 80 - ddboost_plugin_perf 81 82 - name: GPDB7 83 jobs: 84 - GPDB7 85 {% if is_prod and "gpbackup-release" != pipeline_name %} 86 - scale-GPDB7 87 {% endif %} 88 89 {% if is_prod and "gpbackup-release" != pipeline_name %} 90 - name: Scale 91 jobs: 92 - scale-43 93 - scale-5x 94 - scale-6x 95 - scale-GPDB7 96 - s3_plugin_perf 97 - ddboost_plugin_perf 98 {% endif %} 99 100 {% if "gpbackup-release" == pipeline_name %} 101 - name: Packaging and Release 102 jobs: 103 - build_binaries 104 - push-to-github 105 - green_gate 106 - push-to-pivnet 107 {% endif %} 108 109 ##### Anchors ##### 110 anchors: 111 - &ccp_default_params 112 action: create 113 delete_on_failure: true 114 generate_random_name: true 115 plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 116 terraform_source: ccp_src/google/ 117 118 - &ccp_default_vars 119 instance_type: n1-standard-1 120 PLATFORM: centos6 121 122 - &ccp_gen_cluster_default_params 123 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 124 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 125 AWS_DEFAULT_REGION: ((dp/aws-region)) 126 BUCKET_PATH: clusters-google/ 127 BUCKET_NAME: ((dp/tf-bucket-name)) 128 CLOUD_PROVIDER: google 129 130 - &destroy_params 131 action: destroy 132 plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 133 terraform_source: ccp_src/google/ 134 env_name_file: terraform/name 135 vars: 136 aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. 137 aws_ebs_volume_type: standard 138 139 - &ccp_destroy 140 put: terraform 141 params: 142 <<: *destroy_params 143 get_params: 144 action: destroy 145 146 - &ccp_destroy_nvme 147 put: terraform 148 params: 149 <<: *destroy_params 150 terraform_source: ccp_src/google-nvme-block-device/ 151 get_params: 152 action: destroy 153 154 - &set_failed 155 do: 156 - task: on_failure_set_failed 157 config: 158 platform: linux 159 image_resource: 160 type: registry-image 161 source: 162 repository: gcr.io/data-gpdb-public-images/ccp 163 tag: "latest" 164 inputs: 165 - name: ccp_src 166 - name: terraform 167 run: 168 path: 'ccp_src/google/ccp_failed_test.sh' 169 params: 170 GOOGLE_CREDENTIALS: ((dp/$$DEV_PROD$$/google-service-account-key)) 171 GOOGLE_PROJECT_ID: ((dp/$$DEV_PROD$$/google-project-id)) 172 GOOGLE_ZONE: ((dp/$$DEV_PROD$$/google-zone)) 173 GOOGLE_SERVICE_ACCOUNT: ((dp/$$DEV_PROD$$/google-service-account)) 174 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 175 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 176 AWS_DEFAULT_REGION: ((dp/tf-machine-region)) 177 BUCKET_PATH: clusters-google/ 178 BUCKET_NAME: ((dp/tf-bucket-name)) 179 180 - &terraform_cluster 181 type: terraform 182 source: 183 env: 184 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 185 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 186 GOOGLE_CREDENTIALS: ((dp/$$DEV_PROD$$/google-service-account-key)) 187 vars: 188 project_id: ((dp/$$DEV_PROD$$/google-project-id)) 189 storage: 190 access_key_id: ((tf-machine-access-key-id)) 191 secret_access_key: ((tf-machine-secret-access-key)) 192 region_name: ((dp/aws-region)) 193 # This is not parameterized, on purpose. All tfstates will go to this spot, 194 # and different teams will place there clusters' tfstate files under different paths 195 bucket: gpdb5-pipeline-dynamic-terraform 196 bucket_path: clusters-google/ 197 198 - &ddboost_params 199 DD_SOURCE_HOST: ((dp/$$DEV_PROD$$/datadomain_source_host_gcp)) 200 DD_USER: ((dp/$$DEV_PROD$$/datadomain_user)) 201 DD_PW: ((dp/$$DEV_PROD$$/datadomain_password_gcp)) 202 DD_DEST_HOST: ((dp/$$DEV_PROD$$/datadomain_dest_host_gcp)) 203 DD_ENCRYPTED_PW: ((dp/$$DEV_PROD$$/encrypted_datadomain_password_gcp)) 204 205 {% if is_prod or "gpbackup-release" == pipeline_name %} 206 - &slack_alert 207 put: slack-alert 208 params: 209 text: | 210 [gpbackup/$BUILD_JOB_NAME] failed: 211 https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME 212 {% endif %} 213 214 resource_types: 215 - name: terraform 216 type: registry-image 217 source: 218 repository: ljfranklin/terraform-resource 219 # We fixed this version after v0.12.1 had a breaking change 220 tag: 0.11.14 221 222 - name: slack-notification 223 type: registry-image 224 source: 225 repository: cfcommunity/slack-notification-resource 226 tag: latest 227 228 - name: gcs 229 type: registry-image 230 source: 231 repository: frodenas/gcs-resource 232 233 - name: pivnet 234 type: registry-image 235 source: 236 repository: pivotalcf/pivnet-resource 237 tag: latest-final 238 239 ############################################## 240 resources: 241 ##### Docker Images ##### 242 - name: centos6-gpdb5-image 243 type: registry-image 244 source: 245 repository: gcr.io/data-gpdb-public-images/gpdb5-centos6-build-test 246 tag: latest 247 248 - name: centos7-gpdb5-image 249 type: registry-image 250 source: 251 repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test 252 tag: latest 253 254 - name: centos6-gpdb6-image 255 type: registry-image 256 source: 257 repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-test 258 tag: latest 259 260 - name: centos7-gpdb6-image 261 type: registry-image 262 source: 263 repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-test 264 tag: latest 265 266 - name: centos7-gpdb6-golang-image 267 type: registry-image 268 source: 269 repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-test-golang 270 tag: latest 271 272 - name: rocky8-gpdb6-image 273 type: registry-image 274 source: 275 repository: gcr.io/data-gpdb-public-images/gpdb6-rocky8-test 276 tag: latest 277 278 - name: centos7-gpdb7-image 279 type: registry-image 280 source: 281 repository: gcr.io/data-gpdb-public-images/gpdb7-centos7-test 282 tag: latest 283 284 - name: rocky8-gpdb7-image 285 type: registry-image 286 source: 287 repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test 288 tag: latest 289 290 {% if is_prod or "gpbackup-release" == pipeline_name %} 291 - name: sles11-image 292 type: registry-image 293 source: 294 repository: gcr.io/data-gpdb-private-images/gpdb5-sles11-build-test 295 tag: latest 296 username: _json_key 297 password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) 298 299 - name: ubuntu-debian-image 300 type: registry-image 301 source: 302 repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build 303 tag: latest 304 305 - name: ubuntu-debian-test-image 306 type: registry-image 307 source: 308 repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test 309 tag: latest 310 {% endif %} 311 312 ############################################## 313 314 ##### Source Code ##### 315 # TODO mark these as src with name change 316 - name: gpbackup 317 type: git 318 icon: github-circle 319 source: 320 uri: https://github.com/cloudberrydb/gpbackup 321 branch: ((gpbackup-git-branch)) 322 {% if "gpbackup-release" == pipeline_name %} 323 tag_filter: 1.* 324 {% endif %} 325 326 - name: gpbackup_s3_plugin 327 type: git 328 icon: github-circle 329 source: 330 branch: ((dp/$$DEV_PROD$$/gpbackup-s3-plugin-git-branch)) 331 uri: https://github.com/cloudberrydb/gpbackup-s3-plugin 332 {% if "gpbackup-release" == pipeline_name %} 333 tag_filter: 1.* 334 {% endif %} 335 336 - name: gpbackup_ddboost_plugin 337 type: git 338 icon: github-circle 339 source: 340 branch: ((dp/gpbackup-ddboost-plugin-branch)) 341 private_key: ((dp/gpbackup-ddboost-plugin-remote-key)) 342 uri: ((dp/gpbackup-ddboost-plugin-git-remote)) 343 {% if "gpbackup-release" == pipeline_name %} 344 tag_filter: 1.* 345 {% endif %} 346 347 - name: gpbackup_manager_src 348 type: git 349 icon: github-circle 350 source: 351 branch: main 352 private_key: ((dp/$$DEV_PROD$$/gp-backup-manager-remote-deploy-key)) 353 uri: ((dp/gp-backup-manager-git-remote)) 354 {% if "gpbackup-release" == pipeline_name %} 355 tag_filter: 1.* 356 {% endif %} 357 358 - name: gpdb5_src 359 type: git 360 icon: github-circle 361 source: 362 uri: https://github.com/greenplum-db/gpdb 363 branch: 5X_STABLE 364 {% if "gpbackup-release" == pipeline_name %} 365 tag_filter: 5.* 366 {% endif %} 367 368 - name: gpdb6_src 369 type: git 370 icon: github-circle 371 source: 372 uri: https://github.com/greenplum-db/gpdb 373 branch: 6X_STABLE 374 {% if "gpbackup-release" == pipeline_name %} 375 tag_filter: 6.* 376 {% endif %} 377 378 - name: gpdb_main_src 379 type: git 380 icon: github-circle 381 source: 382 uri: https://github.com/greenplum-db/gpdb 383 branch: main 384 385 - name: ccp_src 386 type: git 387 icon: github-circle 388 source: 389 branch: ((dp/ccp-git-branch)) 390 private_key: ((gp-concourse-cluster-provisioner-git-key)) 391 uri: ((dp/ccp-git-remote)) 392 393 ############################################## 394 395 ##### Binaries ##### 396 {% if "gpbackup-release" != pipeline_name %} 397 # We use this intermediate binary in an effort to catch possible 398 # inconsistencies (between a new gpdb5 bin and a fixed version of gpbackup) 399 # before a final release_candidate is generated & shipped 400 - name: bin_gpdb_5x_stable_intermediate 401 type: s3 402 icon: amazon 403 source: 404 bucket: gpdb5-stable-concourse-builds 405 versioned_file: bin_gpdb_centos/bin_gpdb.tar.gz 406 region_name: us-west-2 407 access_key_id: ((aws-bucket-access-key-id)) 408 secret_access_key: ((aws-bucket-secret-access-key)) 409 {% endif %} 410 411 {% if is_prod or "gpbackup-release" == pipeline_name %} 412 - name: bin_gpdb_5x_sles11 413 type: s3 414 icon: amazon 415 source: 416 bucket: gpdb5-stable-concourse-builds 417 access_key_id: ((aws-bucket-access-key-id)) 418 region_name: ((dp/aws-region)) 419 secret_access_key: ((aws-bucket-secret-access-key)) 420 versioned_file: bin_gpdb_sles11/gpdb_branch_5X_STABLE/icw_green/bin_gpdb.tar.gz 421 422 - name: bin_gpdb_6x_stable_ubuntu 423 type: gcs 424 icon: google 425 source: 426 bucket: ((dp/$$DEV_PROD$$/gcs-bucket)) 427 json_key: ((concourse-gcs-resources-service-account-key)) 428 regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64((dp/$$DEV_PROD$$/rc-build-type-gcs)).tar.gz 429 {% endif %} 430 431 - name: bin_gpdb_7x_rhel8 432 type: gcs 433 icon: google 434 source: 435 bucket: ((dp/$$DEV_PROD$$/gcs-bucket)) 436 # bucket: pivotal-gpdb-concourse-resources-intermediates-prod 437 json_key: ((concourse-gcs-resources-service-account-key)) 438 regexp: server/published/main/server-rc-(.*)-rhel8_x86_64.tar.gz 439 # versioned_file: gpdb_main_rhel8/bin_gpdb_rhel8/bin_gpdb.tar.gz 440 441 - name: bin_gpdb_7x_centos7 442 type: gcs 443 icon: google 444 source: 445 bucket: ((dp/$$DEV_PROD$$/gcs-bucket)) 446 json_key: ((concourse-gcs-resources-service-account-key)) 447 regexp: server/published/master/server-rc-(.*)-rhel7_x86_64((dp/$$DEV_PROD$$/rc-build-type-gcs)).tar.gz 448 449 - name: bin_gpdb_5x_stable_centos6 450 type: s3 451 icon: amazon 452 source: 453 bucket: gpdb-stable-concourse-builds 454 versioned_file: release_candidates/bin_gpdb_centos6/gpdb5/bin_gpdb.tar.gz 455 region_name: us-west-2 456 access_key_id: ((aws-bucket-access-key-id)) 457 secret_access_key: ((aws-bucket-secret-access-key)) 458 459 - name: bin_gpdb_5x_stable_centos7 460 type: s3 461 icon: amazon 462 source: 463 bucket: gpdb-stable-concourse-builds 464 versioned_file: release_candidates/bin_gpdb_centos7/gpdb5/bin_gpdb.tar.gz 465 region_name: us-west-2 466 access_key_id: ((aws-bucket-access-key-id)) 467 secret_access_key: ((aws-bucket-secret-access-key)) 468 469 - name: bin_gpdb_6x_stable_centos6 470 type: gcs 471 icon: google 472 source: 473 bucket: ((dp/$$DEV_PROD$$/gcs-bucket)) 474 json_key: ((concourse-gcs-resources-service-account-key)) 475 regexp: server/published/gpdb6/server-rc-(.*)-rhel6_x86_64((dp/$$DEV_PROD$$/rc-build-type-gcs)).tar.gz 476 477 - name: bin_gpdb_6x_stable_centos7 478 type: gcs 479 icon: google 480 source: 481 bucket: ((dp/$$DEV_PROD$$/gcs-bucket)) 482 json_key: ((concourse-gcs-resources-service-account-key)) 483 regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64((dp/$$DEV_PROD$$/rc-build-type-gcs)).tar.gz 484 485 - name: bin_gpdb_6x_rhel8 486 type: gcs 487 source: 488 bucket: ((dp/$$DEV_PROD$$/gcs-bucket)) 489 json_key: ((concourse-gcs-resources-service-account-key)) 490 regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64((dp/$$DEV_PROD$$/rc-build-type-gcs)).tar.gz 491 492 - name: bin_gpdb_43_stable 493 type: s3 494 icon: amazon 495 source: 496 bucket: gpdb-4.3-stable-concourse 497 versioned_file: bin_gpdb_centos/bin_gpdb.tar.gz 498 region_name: us-west-2 499 access_key_id: ((gpdb4-bucket-access-key-id)) 500 secret_access_key: ((gpdb4-bucket-secret-access-key)) 501 502 # These binaries are used for backwards-compatibility testing only 503 - name: bin_gpbackup_1.0.0_and_1.7.1 504 type: s3 505 icon: amazon 506 source: 507 bucket: gpbackup-dependencies 508 versioned_file: gpbackup_bins_1.0.0_and_1.7.1.tar.gz 509 region_name: us-west-2 510 access_key_id: ((aws-bucket-access-key-id)) 511 secret_access_key: ((aws-bucket-secret-access-key)) 512 513 ############################################## 514 515 ##### Other Resources ##### 516 {% if (nightly_trigger and "gpbackup-release" != pipeline_name) %} 517 - name: nightly-trigger 518 type: time 519 source: 520 location: America/Los_Angeles 521 days: [Sunday, Monday, Tuesday, Wednesday, Thursday, Friday] 522 start: 6:00 AM 523 stop: 7:00 AM 524 {% endif %} 525 526 {% if is_prod or "gpbackup-release" == pipeline_name %} 527 # This is specifically for sles11 images because it cannot connect to github due to TLS issues 528 - name: libyaml-0.1.7 529 type: s3 530 icon: amazon 531 source: 532 bucket: gpbackup-dependencies 533 versioned_file: gpbackup-dependencies/libyaml-0.1.7.tar.gz 534 region_name: us-west-2 535 access_key_id: ((aws-bucket-access-key-id)) 536 secret_access_key: ((aws-bucket-secret-access-key)) 537 538 - name: pgcrypto43 539 type: s3 540 icon: amazon 541 source: 542 bucket: pgcrypto43 543 access_key_id: ((aws-bucket-access-key-id)) 544 region_name: ((dp/aws-region)) 545 secret_access_key: ((aws-bucket-secret-access-key)) 546 regexp: pgcrypto-ossv1.1_pv(.*)_gpdb4.3orca-rhel5-x86_64.gppkg 547 {% endif %} 548 549 {% if "gpbackup-release" != pipeline_name %} 550 # Manual caching to prevent dep flakes when downloading dependencies 551 - name: gpbackup_1.12.1_dependencies 552 type: s3 553 icon: amazon 554 source: 555 bucket: gpbackup-dependencies 556 versioned_file: gpbackup_1.12.1_dependencies.tar.gz 557 region_name: us-west-2 558 access_key_id: ((aws-bucket-access-key-id)) 559 secret_access_key: ((aws-bucket-secret-access-key)) 560 {% endif %} 561 562 - name: dummy_seclabel_linux_gpdb6 563 type: gcs 564 icon: google 565 source: 566 bucket: dummy_seclabel_gpdb_linux 567 json_key: ((dp/$$DEV_PROD$$/gcp_svc_acct_key)) 568 regexp: dummy_seclabel_gpdb6-v(.*).so 569 570 - name: dummy_seclabel_linux_main 571 type: gcs 572 icon: google 573 source: 574 bucket: dummy_seclabel_gpdb_linux 575 json_key: ((dp/$$DEV_PROD$$/gcp_svc_acct_key)) 576 regexp: dummy_seclabel_gpdb7-v(.*).so 577 578 - name: terraform 579 <<: *terraform_cluster 580 581 - name: terraform.d 582 type: s3 583 icon: amazon 584 source: 585 access_key_id: ((aws-bucket-access-key-id)) 586 bucket: ccp-terraform-provider-plugins 587 region_name: ((dp/aws-region)) 588 secret_access_key: ((aws-bucket-secret-access-key)) 589 versioned_file: plugin-cache-all.tgz 590 591 {% if is_prod and "gpbackup-release" != pipeline_name %} 592 - name: scale_schema 593 type: s3 594 icon: amazon 595 source: 596 access_key_id: ((gpdb4-bucket-access-key-id)) 597 bucket: ((dp/bucket)) 598 region_name: ((dp/aws-region)) 599 secret_access_key: ((gpdb4-bucket-secret-access-key)) 600 versioned_file: scale-schemas/scale_db1.tgz 601 {% endif %} 602 603 - name: boostfs_installer 604 type: s3 605 icon: amazon 606 source: 607 access_key_id: ((gpdb4-bucket-access-key-id)) 608 bucket: ((dp/bucket)) 609 region_name: ((dp/aws-region)) 610 secret_access_key: ((gpdb4-bucket-secret-access-key)) 611 versioned_file: DDBoostFS-1.1.0.1-565598.rhel.x86_64.rpm 612 613 {% if is_prod or "gpbackup-release" == pipeline_name %} 614 - name: slack-alert 615 type: slack-notification 616 source: 617 url: ((dp/webhook_url)) 618 disable: ((dp/$$DEV_PROD$$/disable_slack_alert)) 619 {% endif %} 620 621 - name: pivnet_release_cache 622 type: s3 623 icon: amazon 624 source: 625 access_key_id: ((aws-bucket-access-key-id)) 626 bucket: ((dp/$$DEV_PROD$$/pivnet_bucket_name)) 627 region_name: ((dp/aws-region)) 628 secret_access_key: ((aws-bucket-secret-access-key)) 629 regexp: pivnet_release_version/v-(.*) 630 631 {% if "gpbackup-release" == pipeline_name %} 632 - name: github_release_components 633 type: s3 634 icon: amazon 635 source: 636 access_key_id: ((aws-bucket-access-key-id)) 637 bucket: ((dp/$$DEV_PROD$$/gpdb-stable-bucket-name)) 638 region_name: ((dp/aws-region)) 639 secret_access_key: ((aws-bucket-secret-access-key)) 640 regexp: components/gpbackup/intermediates/gpbackup-(.*).tar.gz 641 642 - name: github_release_components_rc 643 type: s3 644 icon: amazon 645 source: 646 access_key_id: ((aws-bucket-access-key-id)) 647 bucket: ((dp/$$DEV_PROD$$/gpdb-stable-bucket-name)) 648 region_name: ((dp/aws-region)) 649 secret_access_key: ((aws-bucket-secret-access-key)) 650 regexp: components/gpbackup/gpbackup-(.*).tar.gz 651 652 - name: gppkgs_rc 653 type: s3 654 icon: amazon 655 source: 656 access_key_id: ((aws-bucket-access-key-id)) 657 bucket: ((dp/$$DEV_PROD$$/gpdb-stable-bucket-name)) 658 region_name: ((dp/aws-region)) 659 secret_access_key: ((aws-bucket-secret-access-key)) 660 versioned_file: gppkgs/gpbackup-gppkgs.tar.gz 661 {% endif %} 662 663 - name: gpbackup-go-components 664 type: s3 665 icon: amazon 666 source: 667 access_key_id: ((aws-bucket-access-key-id)) 668 bucket: ((dp/$$DEV_PROD$$/gpdb-stable-bucket-name)) 669 region_name: ((dp/aws-region)) 670 secret_access_key: ((aws-bucket-secret-access-key)) 671 versioned_file: gpbackup-go-components/go_components.tar.gz 672 673 - name: gppkgs 674 type: s3 675 icon: amazon 676 source: 677 access_key_id: ((aws-bucket-access-key-id)) 678 bucket: ((dp/$$DEV_PROD$$/gpdb-stable-bucket-name)) 679 region_name: ((dp/aws-region)) 680 secret_access_key: ((aws-bucket-secret-access-key)) 681 {% if "gpbackup-release" == pipeline_name %} 682 versioned_file: gppkgs/intermediates_release/gpbackup-gppkgs.tar.gz 683 {% else %} 684 versioned_file: gppkgs/intermediates/gpbackup-gppkgs.tar.gz 685 {% endif %} 686 687 {% if "gpbackup-release" == pipeline_name %} 688 - name: gpbackup_github_release 689 type: github-release 690 source: 691 owner: greenplum-db 692 repository: gpbackup 693 access_token: ((dp/$$DEV_PROD$$/gpbackup-git-access-token)) 694 release: true 695 {% endif %} 696 697 {% if "gpbackup-release" == pipeline_name %} 698 - name: pivnet-upload 699 type: pivnet 700 source: 701 api_token: ((dp/$$DEV_PROD$$/pivnet_api_token)) 702 product_slug: pivotal-gpdb-backup-restore 703 endpoint: "https://network.tanzu.vmware.com" 704 bucket: gpbackup-pivotal-network 705 access_key_id: ((dp/$$DEV_PROD$$/pivnet_bucket_access_key_id)) 706 secret_access_key: ((dp/$$DEV_PROD$$/pivnet_bucket_secret_access_key)) 707 region: us-west-2 708 copy_metadata: true 709 sort_by: semver 710 {% endif %} 711 712 - name: gpbackup-release-license 713 type: gcs 714 icon: google 715 source: 716 bucket: gpbackup-release-licenses 717 json_key: ((dp/$$DEV_PROD$$/gcp_svc_acct_key)) 718 regexp: open_source_license_VMware_Tanzu_Greenplum_Backup_and_Restore_(.*)_.*.txt 719 720 ############################################## 721 722 723 jobs: 724 - name: build_binaries 725 plan: 726 - in_parallel: 727 {% if "gpbackup-release" != pipeline_name %} 728 - get: gpbackup_s3_plugin 729 trigger: true 730 - get: gpbackup_manager_src 731 trigger: true 732 - get: gpbackup 733 trigger: true 734 {% else %} 735 - get: gpbackup_s3_plugin 736 - get: gpbackup_manager_src 737 - get: gpbackup 738 {% endif %} 739 - get: pivnet_release_cache 740 - task: build-go-binaries 741 file: gpbackup/ci/tasks/build-go-binaries.yml 742 - put: gpbackup-go-components 743 params: 744 file: go_components/go_components.tar.gz 745 746 - name: build_gppkgs 747 plan: 748 - in_parallel: 749 - get: centos6-gpdb5-image 750 {% if is_prod or "gpbackup-release" == pipeline_name %} 751 - get: sles11-image 752 - get: bin_gpdb_5x_sles11 753 - get: libyaml-0.1.7 754 - get: ubuntu-debian-image 755 - get: ubuntu-debian-test-image 756 - get: bin_gpdb_6x_stable_ubuntu 757 {% endif %} 758 - get: gpdb_src 759 resource: gpdb6_src 760 - get: gpbackup-go-components 761 trigger: true 762 passed: [build_binaries] 763 - get: bin_gpdb_6x_stable_centos6 764 - get: gpbackup 765 passed: [build_binaries] 766 - get: gpbackup_ddboost_plugin 767 {% if "gpbackup-release" != pipeline_name %} 768 trigger: true 769 {% endif %} 770 - get: pivnet_release_cache 771 - get: gpbackup-release-license 772 - task: gpbackup-tools-versions 773 image: centos6-gpdb5-image 774 file: gpbackup/ci/tasks/gpbackup-tools-versions.yml 775 - in_parallel: 776 - do: # RHEL 777 - task: build-ddboost-RHEL 778 image: centos6-gpdb5-image 779 file: gpbackup/ci/tasks/build-ddboost.yml 780 input_mapping: 781 bin_gpdb: bin_gpdb_6x_stable_centos6 782 - task: tar-binaries-RHEL 783 image: centos6-gpdb5-image 784 file: gpbackup/ci/tasks/build-os-tars.yml 785 - task: build_gppkgs-RHEL 786 image: centos6-gpdb5-image 787 file: gpbackup/ci/tasks/build-gppkg.yml 788 input_mapping: 789 bin_gpdb: bin_gpdb_6x_stable_centos6 790 output_mapping: 791 gppkgs: rhel-gppkg 792 params: 793 OS: RHEL 794 {% if is_prod or "gpbackup-release" == pipeline_name %} 795 - do: # SLES 796 - task: build-ddboost-SLES 797 image: sles11-image 798 file: gpbackup/ci/tasks/build-ddboost.yml 799 input_mapping: 800 bin_gpdb: bin_gpdb_5x_sles11 801 - task: tar-binaries-SLES 802 image: sles11-image 803 file: gpbackup/ci/tasks/build-os-tars.yml 804 - task: build_gppkgs-SLES 805 # We need to use centos image for doing rpm-build for sles 806 image: centos6-gpdb5-image 807 file: gpbackup/ci/tasks/build-gppkg.yml 808 input_mapping: 809 bin_gpdb: bin_gpdb_5x_sles11 810 output_mapping: 811 gppkgs: sles-gppkg 812 params: 813 OS: SLES 814 - do: # ubuntu 815 - task: build-ddboost-ubuntu 816 image: ubuntu-debian-image 817 file: gpbackup/ci/tasks/build-ddboost.yml 818 input_mapping: 819 bin_gpdb: bin_gpdb_6x_stable_ubuntu 820 - task: tar-binaries-ubuntu 821 image: ubuntu-debian-image 822 file: gpbackup/ci/tasks/build-os-tars.yml 823 - task: build_gppkgs-ubuntu 824 image: ubuntu-debian-test-image 825 file: gpbackup/ci/tasks/build-gppkg.yml 826 input_mapping: 827 bin_gpdb: bin_gpdb_6x_stable_ubuntu 828 output_mapping: 829 gppkgs: ubuntu-gppkg 830 params: 831 OS: ubuntu 832 {% endif %} 833 - task: tar-gppkgs 834 image: centos6-gpdb5-image 835 file: gpbackup/ci/tasks/tar-gppkgs.yml 836 - put: gppkgs 837 params: 838 file: gppkgs/gpbackup-gppkgs.tar.gz 839 {% if "gpbackup-release" == pipeline_name %} 840 - put: github_release_components 841 params: 842 file: gpbackup_tar/gpbackup-*.tar.gz 843 {% endif %} 844 845 # TODO: these should be modernized to gpdb6_src and gpdb6 images when possible 846 - name: s3_plugin_tests 847 plan: 848 - in_parallel: 849 - get: centos7-gpdb5-image 850 - get: gpbackup_s3_plugin 851 passed: [build_binaries] 852 - get: gpbackup 853 passed: [build_gppkgs] 854 - get: gpdb_src 855 resource: gpdb5_src 856 - get: gpdb_binary 857 resource: bin_gpdb_5x_stable_centos6 858 - get: gppkgs 859 passed: [build_gppkgs] 860 trigger: true 861 - do: 862 - get: ccp_src 863 - get: terraform.d 864 params: 865 unpack: true 866 - put: terraform 867 params: 868 <<: *ccp_default_params 869 vars: 870 <<: *ccp_default_vars 871 PLATFORM: centos7 872 - task: gen_cluster 873 file: ccp_src/ci/tasks/gen_cluster.yml 874 params: 875 <<: *ccp_gen_cluster_default_params 876 PLATFORM: centos7 877 - task: gpinitsystem 878 file: ccp_src/ci/tasks/gpinitsystem.yml 879 - task: setup-centos-env 880 image: centos7-gpdb5-image 881 file: gpbackup/ci/tasks/setup-centos-env.yml 882 - task: run_tests 883 image: centos7-gpdb5-image 884 file: gpbackup/ci/tasks/s3-plugin-tests.yml 885 params: 886 REGION: us-west-2 887 AWS_ACCESS_KEY_ID: ((aws-bucket-access-key-id)) 888 AWS_SECRET_ACCESS_KEY: ((aws-bucket-secret-access-key)) 889 BUCKET: ((dp/$$DEV_PROD$$/gpbackup-s3-plugin-test-bucket)) 890 on_success: 891 <<: *ccp_destroy 892 {% if is_prod or "gpbackup-release" == pipeline_name %} 893 on_failure: 894 *slack_alert 895 {% endif %} 896 ensure: 897 <<: *set_failed 898 899 - name: s3_plugin_perf 900 plan: 901 - in_parallel: 902 - get: centos7-gpdb6-image 903 - get: gpbackup_s3_plugin 904 passed: [build_binaries] 905 - get: gpbackup 906 passed: [build_gppkgs] 907 - get: gpdb_src 908 resource: gpdb6_src 909 - get: gpdb_binary 910 resource: bin_gpdb_6x_stable_centos6 911 - get: gppkgs 912 passed: [build_gppkgs] 913 trigger: true 914 - do: 915 - get: ccp_src 916 - get: terraform.d 917 params: 918 unpack: true 919 - put: terraform 920 params: 921 <<: *ccp_default_params 922 terraform_source: ccp_src/google-nvme-block-device/ 923 vars: 924 PLATFORM: centos6 925 number_of_nodes: 4 926 segments_per_host: 4 927 instance_type: n1-standard-8 928 ccp_reap_minutes: 360 929 - task: gen_cluster 930 file: ccp_src/ci/tasks/gen_cluster.yml 931 params: 932 <<: *ccp_gen_cluster_default_params 933 - task: gpinitsystem 934 file: ccp_src/ci/tasks/gpinitsystem.yml 935 - task: setup-centos-env 936 image: centos7-gpdb6-image 937 file: gpbackup/ci/tasks/setup-centos-env.yml 938 - task: run_perf 939 image: centos7-gpdb6-image 940 file: gpbackup/ci/tasks/s3-plugin-perf.yml 941 params: 942 REGION: us-west-2 943 AWS_ACCESS_KEY_ID: ((aws-bucket-access-key-id)) 944 AWS_SECRET_ACCESS_KEY: ((aws-bucket-secret-access-key)) 945 BUCKET: ((dp/$$DEV_PROD$$/gpbackup-s3-plugin-test-bucket)) 946 SCALE_FACTOR: 1 947 on_success: 948 <<: *ccp_destroy 949 {% if is_prod or "gpbackup-release" == pipeline_name %} 950 on_failure: 951 *slack_alert 952 {% endif %} 953 ensure: 954 <<: *set_failed 955 956 - name: gpbackup-manager-tests 957 plan: 958 - in_parallel: 959 - get: centos7-gpdb6-golang-image 960 - get: gpbackup_manager_src 961 passed: [build_binaries] 962 - get: gpbackup 963 passed: [build_gppkgs] 964 - get: bin_gpdb 965 resource: bin_gpdb_6x_stable_centos6 966 - get: gpdb_src 967 resource: gpdb6_src 968 - get: gppkgs 969 trigger: true 970 passed: [build_gppkgs] 971 - task: run_tests 972 image: centos7-gpdb6-golang-image 973 file: gpbackup/ci/tasks/test-gpbackup-manager.yml 974 975 {% if is_prod or "gpbackup-release" == pipeline_name %} 976 - name: ddboost_plugin_and_boostfs_tests_43 977 plan: 978 - in_parallel: 979 - get: gpdb_src 980 resource: gpdb5_src 981 - get: gpdb_binary 982 resource: bin_gpdb_43_stable 983 - get: pgcrypto43 984 - get: centos6-gpdb5-image 985 - get: gpbackup_ddboost_plugin 986 passed: [build_gppkgs] 987 - get: gpbackup 988 passed: [build_gppkgs] 989 - get: boostfs_installer 990 - get: ccp_src 991 - get: gppkgs 992 passed: [build_gppkgs] 993 trigger: true 994 - get: terraform.d 995 params: 996 unpack: true 997 - put: terraform 998 params: 999 <<: *ccp_default_params 1000 terraform_source: ccp_src/google-nvme-block-device/ 1001 vars: 1002 instance_type: n1-standard-8 1003 PLATFORM: centos6 1004 - task: gen_cluster 1005 params: 1006 <<: *ccp_gen_cluster_default_params 1007 PLATFORM: centos6 1008 file: ccp_src/ci/tasks/gen_cluster.yml 1009 - task: gpinitsystem 1010 file: ccp_src/ci/tasks/gpinitsystem.yml 1011 - task: setup-centos-env 1012 file: gpbackup/ci/tasks/setup-centos-env.yml 1013 image: centos6-gpdb5-image 1014 - task: install-boostfs 1015 file: gpbackup/ci/tasks/boostfs-install.yml 1016 image: centos6-gpdb5-image 1017 params: 1018 <<: *ddboost_params 1019 - task: run_tests 1020 image: centos6-gpdb5-image 1021 file: gpbackup/ci/tasks/ddboost-plugin-tests.yml 1022 params: 1023 <<: *ddboost_params 1024 on_success: 1025 <<: *ccp_destroy_nvme 1026 ensure: 1027 <<: *set_failed 1028 1029 - name: ddboost_plugin_and_boostfs_tests_5x 1030 plan: 1031 - in_parallel: 1032 - get: gpdb_src 1033 resource: gpdb5_src 1034 - get: gpdb_binary 1035 resource: bin_gpdb_5x_stable_centos6 1036 - get: centos6-gpdb5-image 1037 - get: gpbackup_ddboost_plugin 1038 passed: [build_gppkgs] 1039 - get: gpbackup 1040 passed: [build_gppkgs] 1041 - get: boostfs_installer 1042 - get: ccp_src 1043 - get: gppkgs 1044 passed: [build_gppkgs] 1045 trigger: true 1046 - get: terraform.d 1047 params: 1048 unpack: true 1049 - put: terraform 1050 params: 1051 <<: *ccp_default_params 1052 terraform_source: ccp_src/google-nvme-block-device/ 1053 vars: 1054 instance_type: n1-standard-8 1055 PLATFORM: centos6 1056 - task: gen_cluster 1057 params: 1058 <<: *ccp_gen_cluster_default_params 1059 PLATFORM: centos6 1060 file: ccp_src/ci/tasks/gen_cluster.yml 1061 - task: gpinitsystem 1062 file: ccp_src/ci/tasks/gpinitsystem.yml 1063 - task: setup-centos-env 1064 file: gpbackup/ci/tasks/setup-centos-env.yml 1065 image: centos6-gpdb5-image 1066 - task: install-boostfs 1067 file: gpbackup/ci/tasks/boostfs-install.yml 1068 image: centos6-gpdb5-image 1069 params: 1070 <<: *ddboost_params 1071 - task: run_tests 1072 image: centos6-gpdb5-image 1073 file: gpbackup/ci/tasks/ddboost-plugin-tests.yml 1074 params: 1075 <<: *ddboost_params 1076 on_success: 1077 <<: *ccp_destroy_nvme 1078 ensure: 1079 <<: *set_failed 1080 {% endif %} 1081 1082 - name: ddboost_plugin_and_boostfs_tests_6x 1083 plan: 1084 - in_parallel: 1085 - get: gpdb_src 1086 resource: gpdb6_src 1087 - get: gpdb_binary 1088 resource: bin_gpdb_6x_stable_centos6 1089 - get: dummy_seclabel 1090 resource: dummy_seclabel_linux_gpdb6 1091 - get: centos6-gpdb6-image 1092 - get: gpbackup_ddboost_plugin 1093 passed: [build_gppkgs] 1094 - get: gpbackup 1095 passed: [build_gppkgs] 1096 - get: boostfs_installer 1097 - get: ccp_src 1098 - get: gppkgs 1099 passed: [build_gppkgs] 1100 trigger: true 1101 - get: terraform.d 1102 params: 1103 unpack: true 1104 - put: terraform 1105 params: 1106 <<: *ccp_default_params 1107 terraform_source: ccp_src/google-nvme-block-device/ 1108 vars: 1109 instance_type: n1-standard-8 1110 PLATFORM: centos6 1111 - task: gen_cluster 1112 params: 1113 <<: *ccp_gen_cluster_default_params 1114 PLATFORM: centos6 1115 file: ccp_src/ci/tasks/gen_cluster.yml 1116 - task: gpinitsystem 1117 file: ccp_src/ci/tasks/gpinitsystem.yml 1118 - task: setup-centos-env 1119 file: gpbackup/ci/tasks/setup-centos-env.yml 1120 image: centos6-gpdb6-image 1121 - task: install-boostfs 1122 file: gpbackup/ci/tasks/boostfs-install.yml 1123 image: centos6-gpdb6-image 1124 params: 1125 <<: *ddboost_params 1126 - task: run_tests 1127 image: centos6-gpdb6-image 1128 file: gpbackup/ci/tasks/ddboost-plugin-tests.yml 1129 params: 1130 <<: *ddboost_params 1131 on_success: 1132 <<: *ccp_destroy_nvme 1133 ensure: 1134 <<: *set_failed 1135 1136 - name: ddboost_plugin_perf 1137 plan: 1138 - in_parallel: 1139 - get: centos7-gpdb6-image 1140 - get: gpbackup_ddboost_plugin 1141 passed: [build_gppkgs] 1142 - get: gpbackup 1143 passed: [build_gppkgs] 1144 - get: boostfs_installer 1145 - get: ccp_src 1146 - get: gpdb_src 1147 resource: gpdb6_src 1148 - get: gpdb_binary 1149 resource: bin_gpdb_6x_stable_centos7 1150 - get: gppkgs 1151 passed: [build_gppkgs] 1152 trigger: true 1153 - get: terraform.d 1154 params: 1155 unpack: true 1156 - put: terraform 1157 params: 1158 <<: *ccp_default_params 1159 terraform_source: ccp_src/google-nvme-block-device/ 1160 vars: 1161 instance_type: n1-standard-8 1162 number_of_nodes: 4 1163 segments_per_host: 4 1164 PLATFORM: centos7 1165 - task: gen_cluster 1166 params: 1167 <<: *ccp_gen_cluster_default_params 1168 PLATFORM: centos7 1169 file: ccp_src/ci/tasks/gen_cluster.yml 1170 - task: gpinitsystem 1171 file: ccp_src/ci/tasks/gpinitsystem.yml 1172 - task: setup-centos-env 1173 image: centos7-gpdb6-image 1174 file: gpbackup/ci/tasks/setup-centos-env.yml 1175 - task: run_perf 1176 image: centos7-gpdb6-image 1177 file: gpbackup/ci/tasks/ddboost-plugin-perf.yml 1178 params: 1179 <<: *ddboost_params 1180 REGION: us-west-2 1181 AWS_ACCESS_KEY_ID: ((aws-bucket-access-key-id)) 1182 AWS_SECRET_ACCESS_KEY: ((aws-bucket-secret-access-key)) 1183 BUCKET: ((dp/$$DEV_PROD$$/gpbackup-s3-plugin-test-bucket)) 1184 SCALE_FACTOR: 1 1185 on_success: 1186 <<: *ccp_destroy_nvme 1187 ensure: 1188 <<: *set_failed 1189 1190 {% if is_prod or "gpbackup-release" == pipeline_name %} 1191 - name: GPDB5-sles11 1192 plan: 1193 - in_parallel: 1194 - get: sles11-image 1195 - get: gpbackup 1196 passed: [build_gppkgs] 1197 - get: gpdb_binary 1198 resource: bin_gpdb_5x_sles11 1199 {% if "gpbackup-release" != pipeline_name %} 1200 trigger: true 1201 {% endif %} 1202 - get: ccp_src 1203 - get: gpdb_src 1204 resource: gpdb5_src 1205 - get: gppkgs 1206 passed: [build_gppkgs] 1207 trigger: true 1208 - get: terraform.d 1209 params: 1210 unpack: true 1211 - put: terraform 1212 params: 1213 <<: *ccp_default_params 1214 vars: 1215 <<: *ccp_default_vars 1216 PLATFORM: sles12 1217 default_image_user: root 1218 - task: gen_cluster 1219 params: 1220 <<: *ccp_gen_cluster_default_params 1221 PLATFORM: sles12 1222 file: ccp_src/ci/tasks/gen_cluster.yml 1223 - task: gpinitsystem 1224 file: ccp_src/ci/tasks/gpinitsystem.yml 1225 - task: integration-tests 1226 image: sles11-image 1227 file: gpbackup/ci/tasks/gpbackup-tests.yml 1228 params: 1229 OS: SLES 1230 on_success: 1231 <<: *ccp_destroy 1232 on_failure: 1233 *slack_alert 1234 ensure: 1235 <<: *set_failed 1236 1237 - name: GPDB6-ubuntu 1238 plan: 1239 - in_parallel: 1240 - get: gpbackup 1241 passed: [build_gppkgs] 1242 - get: gpdb_binary 1243 resource: bin_gpdb_6x_stable_ubuntu 1244 {% if "gpbackup-release" != pipeline_name %} 1245 trigger: true 1246 {% endif %} 1247 - get: ccp_src 1248 - get: gpdb_src 1249 resource: gpdb6_src 1250 - get: dummy_seclabel 1251 resource: dummy_seclabel_linux_gpdb6 1252 - get: ubuntu-debian-test-image 1253 - get: gppkgs 1254 passed: [build_gppkgs] 1255 trigger: true 1256 - get: terraform.d 1257 params: 1258 unpack: true 1259 - put: terraform 1260 params: 1261 <<: *ccp_default_params 1262 vars: 1263 <<: *ccp_default_vars 1264 PLATFORM: ubuntu18.04 1265 default_image_user: root 1266 - task: gen_cluster 1267 params: 1268 <<: *ccp_gen_cluster_default_params 1269 PLATFORM: ubuntu 1270 file: ccp_src/ci/tasks/gen_cluster.yml 1271 - task: gpinitsystem 1272 file: ccp_src/ci/tasks/gpinitsystem.yml 1273 - task: integration-tests 1274 image: ubuntu-debian-test-image 1275 file: gpbackup/ci/tasks/gpbackup-tests.yml 1276 params: 1277 OS: ubuntu 1278 default_ami_user: ubuntu 1279 on_success: 1280 <<: *ccp_destroy 1281 on_failure: 1282 *slack_alert 1283 ensure: 1284 <<: *set_failed 1285 {% endif %} 1286 1287 - name: GPDB7 1288 plan: 1289 - in_parallel: 1290 - get: centos7-gpdb7-image 1291 - get: rocky8-gpdb7-image 1292 - get: gpbackup 1293 passed: [build_gppkgs] 1294 - get: bin_gpdb_7x_centos7 1295 trigger: true 1296 - get: bin_gpdb_7x_rhel8 1297 - get: gpdb_src 1298 resource: gpdb_main_src 1299 - get: dummy_seclabel 1300 resource: dummy_seclabel_linux_main 1301 - get: gppkgs 1302 trigger: true 1303 passed: [build_gppkgs] 1304 - in_parallel: 1305 - task: run-tests-locally-centos7 1306 image: centos7-gpdb7-image 1307 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1308 params: 1309 REQUIRES_DUMMY_SEC: true 1310 input_mapping: 1311 bin_gpdb: bin_gpdb_7x_centos7 1312 - task: run-tests-locally-rocky8 1313 image: rocky8-gpdb7-image 1314 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1315 params: 1316 REQUIRES_DUMMY_SEC: true 1317 input_mapping: 1318 bin_gpdb: bin_gpdb_7x_rhel8 1319 {% if is_prod or "gpbackup-release" == pipeline_name %} 1320 on_failure: 1321 *slack_alert 1322 {% endif %} 1323 1324 - name: GPDB4.3 1325 plan: 1326 - in_parallel: 1327 - get: centos6-gpdb5-image 1328 - get: gpbackup 1329 passed: [build_gppkgs] 1330 - get: bin_gpdb 1331 resource: bin_gpdb_43_stable 1332 {% if "gpbackup-release" != pipeline_name %} 1333 trigger: true 1334 {% endif %} 1335 - get: gpdb_src 1336 resource: gpdb5_src 1337 - get: gppkgs 1338 passed: [build_gppkgs] 1339 trigger: true 1340 - task: run-tests-locally-centos6 1341 image: centos6-gpdb5-image 1342 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1343 {% if is_prod or "gpbackup-release" == pipeline_name %} 1344 on_failure: 1345 *slack_alert 1346 {% endif %} 1347 1348 - name: GPDB5 1349 plan: 1350 - in_parallel: 1351 - get: centos6-gpdb5-image 1352 - get: centos7-gpdb5-image 1353 - get: gpbackup 1354 passed: [build_gppkgs] 1355 - get: bin_gpdb_5x_stable_centos6 1356 {% if "gpbackup-release" != pipeline_name %} 1357 trigger: true 1358 {% endif %} 1359 - get: bin_gpdb_5x_stable_centos7 1360 - get: gpdb_src 1361 resource: gpdb5_src 1362 - get: gppkgs 1363 trigger: true 1364 passed: [build_gppkgs] 1365 - in_parallel: 1366 - task: run-tests-locally-centos6 1367 image: centos6-gpdb5-image 1368 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1369 input_mapping: 1370 bin_gpdb: bin_gpdb_5x_stable_centos6 1371 - task: run-tests-locally-centos7 1372 image: centos7-gpdb5-image 1373 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1374 input_mapping: 1375 bin_gpdb: bin_gpdb_5x_stable_centos7 1376 {% if is_prod or "gpbackup-release" == pipeline_name %} 1377 on_failure: 1378 *slack_alert 1379 {% endif %} 1380 1381 - name: GPDB6 1382 plan: 1383 - in_parallel: 1384 - get: centos7-gpdb6-image 1385 - get: rocky8-gpdb6-image 1386 {% if is_prod or "gpbackup-release" == pipeline_name %} 1387 - get: ubuntu-debian-test-image 1388 - get: bin_gpdb_6x_stable_ubuntu 1389 {% endif %} 1390 - get: bin_gpdb_6x_rhel8 1391 - get: gpbackup 1392 passed: [build_gppkgs] 1393 - get: bin_gpdb_6x_stable_centos6 1394 {% if "gpbackup-release" != pipeline_name %} 1395 trigger: true 1396 {% endif %} 1397 - get: bin_gpdb_6x_stable_centos7 1398 - get: gpdb_src 1399 resource: gpdb6_src 1400 - get: dummy_seclabel 1401 resource: dummy_seclabel_linux_gpdb6 1402 - get: gppkgs 1403 trigger: true 1404 passed: [build_gppkgs] 1405 - in_parallel: 1406 - task: run-tests-locally-centos7 1407 image: centos7-gpdb6-image 1408 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1409 params: 1410 REQUIRES_DUMMY_SEC: true 1411 input_mapping: 1412 bin_gpdb: bin_gpdb_6x_stable_centos7 1413 - task: run-tests-locally-rocky8 1414 image: rocky8-gpdb6-image 1415 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1416 params: 1417 REQUIRES_DUMMY_SEC: true 1418 input_mapping: 1419 bin_gpdb: bin_gpdb_6x_rhel8 1420 {% if is_prod or "gpbackup-release" == pipeline_name %} 1421 - task: run-tests-locally-ubuntu-debian 1422 image: ubuntu-debian-test-image 1423 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1424 params: 1425 REQUIRES_DUMMY_SEC: true 1426 OS: ubuntu 1427 input_mapping: 1428 bin_gpdb: bin_gpdb_6x_stable_ubuntu 1429 on_failure: 1430 *slack_alert 1431 {% endif %} 1432 1433 - name: GPDB6-7-seg-cluster 1434 plan: 1435 - in_parallel: 1436 - get: centos7-gpdb6-image 1437 - get: rocky8-gpdb6-image 1438 {% if is_prod or "gpbackup-release" == pipeline_name %} 1439 - get: ubuntu-debian-test-image 1440 - get: bin_gpdb_6x_stable_ubuntu 1441 {% endif %} 1442 - get: gpbackup 1443 passed: [build_gppkgs] 1444 - get: bin_gpdb_6x_stable_centos6 1445 {% if "gpbackup-release" != pipeline_name %} 1446 trigger: true 1447 {% endif %} 1448 - get: bin_gpdb_6x_rhel8 1449 - get: bin_gpdb_6x_stable_centos7 1450 - get: gpdb_src 1451 resource: gpdb6_src 1452 - get: dummy_seclabel 1453 resource: dummy_seclabel_linux_gpdb6 1454 - get: gppkgs 1455 trigger: true 1456 passed: [build_gppkgs] 1457 - in_parallel: 1458 - task: run-tests-locally-centos7 1459 image: centos7-gpdb6-image 1460 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1461 params: 1462 REQUIRES_DUMMY_SEC: true 1463 LOCAL_CLUSTER_SIZE: 7 1464 input_mapping: 1465 bin_gpdb: bin_gpdb_6x_stable_centos7 1466 - task: run-tests-locally-rocky8 1467 image: rocky8-gpdb6-image 1468 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1469 params: 1470 REQUIRES_DUMMY_SEC: true 1471 LOCAL_CLUSTER_SIZE: 7 1472 input_mapping: 1473 bin_gpdb: bin_gpdb_6x_rhel8 1474 {% if is_prod or "gpbackup-release" == pipeline_name %} 1475 - task: run-tests-locally-ubuntu-debian 1476 image: ubuntu-debian-test-image 1477 file: gpbackup/ci/tasks/test-on-local-cluster.yml 1478 params: 1479 REQUIRES_DUMMY_SEC: true 1480 LOCAL_CLUSTER_SIZE: 7 1481 OS: ubuntu 1482 input_mapping: 1483 bin_gpdb: bin_gpdb_6x_stable_ubuntu 1484 on_failure: 1485 *slack_alert 1486 {% endif %} 1487 1488 - name: backward-compatibility 1489 plan: 1490 - in_parallel: 1491 - get: centos7-gpdb6-image 1492 - get: gpbackup 1493 passed: [build_gppkgs] 1494 - get: gpdb_binary 1495 resource: bin_gpdb_5x_stable_centos7 1496 {% if "gpbackup-release" != pipeline_name %} 1497 trigger: true 1498 {% endif %} 1499 - get: bin_gpbackup_1.0.0_and_1.7.1 1500 - get: ccp_src 1501 - get: gpdb_src 1502 resource: gpdb5_src 1503 - get: gppkgs 1504 passed: [build_gppkgs] 1505 trigger: true 1506 - get: terraform.d 1507 params: 1508 unpack: true 1509 - put: terraform 1510 params: 1511 <<: *ccp_default_params 1512 vars: 1513 <<: *ccp_default_vars 1514 PLATFORM: centos7 1515 - task: gen_cluster 1516 params: 1517 <<: *ccp_gen_cluster_default_params 1518 PLATFORM: centos7 1519 file: ccp_src/ci/tasks/gen_cluster.yml 1520 - task: gpinitsystem 1521 file: ccp_src/ci/tasks/gpinitsystem.yml 1522 - task: setup-centos-env 1523 file: gpbackup/ci/tasks/setup-centos-env.yml 1524 image: centos7-gpdb6-image 1525 - task: backup-1.0.0-restore-latest 1526 image: centos7-gpdb6-image 1527 file: gpbackup/ci/tasks/all-tests.yml 1528 params: 1529 GPBACKUP_VERSION: "1.0.0" 1530 - task: backup-1.7.1-restore-latest 1531 image: centos7-gpdb6-image 1532 file: gpbackup/ci/tasks/all-tests.yml 1533 params: 1534 GPBACKUP_VERSION: "1.7.1" 1535 on_success: 1536 <<: *ccp_destroy 1537 ensure: 1538 <<: *set_failed 1539 1540 {% if "gpbackup-release" != pipeline_name %} 1541 # Ensure compatibility between a new GPDB5 binary and a fixed version of gpbackup 1542 - name: 5X-head-gpbackup-fixed-test 1543 plan: 1544 - in_parallel: 1545 - get: centos6-gpdb5-image 1546 # NOTE: This is not being used, but is required input for the final_gate 1547 - get: gppkgs 1548 passed: [build_gppkgs] 1549 trigger: true 1550 - get: gpbackup 1551 passed: [build_gppkgs] 1552 - get: bin_gpdb 1553 resource: bin_gpdb_5x_stable_intermediate 1554 trigger: true 1555 - get: gpdb_src 1556 resource: gpdb5_src 1557 - get: gpbackup_1.12.1_dependencies 1558 - task: gpbackup-1.12.1-test 1559 image: centos6-gpdb5-image 1560 file: gpbackup/ci/tasks/integration-tests-fixed-version.yml 1561 params: 1562 GPBACKUP_VERSION: "1.12.1" 1563 {% endif %} 1564 1565 {% if is_prod and "gpbackup-release" != pipeline_name %} 1566 - name: scale-GPDB7 1567 plan: 1568 - in_parallel: 1569 - get: centos7-gpdb7-image 1570 - get: gpbackup 1571 passed: [build_gppkgs] 1572 - get: gpdb_binary 1573 resource: bin_gpdb_7x_rhel8 1574 - get: ccp_src 1575 - get: gpdb_src 1576 resource: gpdb_main_src 1577 - get: scale_schema 1578 {% if nightly_trigger %} 1579 - get: nightly-trigger 1580 trigger: true 1581 {% endif %} 1582 - get: gppkgs 1583 passed: [build_gppkgs] 1584 - get: terraform.d 1585 params: 1586 unpack: true 1587 - put: terraform 1588 params: 1589 <<: *ccp_default_params 1590 terraform_source: ccp_src/google-nvme-block-device/ 1591 vars: 1592 instance_type: n1-standard-8 1593 PLATFORM: rhel8-gpdb7 1594 - task: gen_cluster 1595 params: 1596 <<: *ccp_gen_cluster_default_params 1597 PLATFORM: rhel8-gpdb7 1598 file: ccp_src/ci/tasks/gen_cluster.yml 1599 - task: gpinitsystem 1600 file: ccp_src/ci/tasks/gpinitsystem.yml 1601 - task: setup-centos-env 1602 image: centos7-gpdb7-image 1603 params: 1604 default_ami_user: rhel 1605 file: gpbackup/ci/tasks/setup-centos-env.yml 1606 - task: scale-tests 1607 image: centos7-gpdb7-image 1608 file: gpbackup/ci/tasks/scale-tests.yml 1609 on_success: 1610 <<: *ccp_destroy_nvme 1611 on_failure: 1612 *slack_alert 1613 ensure: 1614 <<: *set_failed 1615 1616 - name: scale-6x 1617 plan: 1618 - in_parallel: 1619 - get: centos7-gpdb6-image 1620 - get: gpbackup 1621 passed: [build_gppkgs] 1622 - get: gpdb_binary 1623 resource: bin_gpdb_6x_stable_centos7 1624 - get: ccp_src 1625 - get: gpdb_src 1626 resource: gpdb6_src 1627 - get: scale_schema 1628 {% if nightly_trigger and "gpbackup-release" != pipeline_name %} 1629 - get: nightly-trigger 1630 trigger: true 1631 {% endif %} 1632 - get: gppkgs 1633 passed: [build_gppkgs] 1634 - get: terraform.d 1635 params: 1636 unpack: true 1637 - put: terraform 1638 params: 1639 <<: *ccp_default_params 1640 terraform_source: ccp_src/google-nvme-block-device/ 1641 vars: 1642 instance_type: n1-standard-8 1643 PLATFORM: centos7 1644 - task: gen_cluster 1645 params: 1646 <<: *ccp_gen_cluster_default_params 1647 PLATFORM: centos7 1648 file: ccp_src/ci/tasks/gen_cluster.yml 1649 - task: gpinitsystem 1650 file: ccp_src/ci/tasks/gpinitsystem.yml 1651 - task: setup-centos-env 1652 image: centos7-gpdb6-image 1653 file: gpbackup/ci/tasks/setup-centos-env.yml 1654 - task: scale-tests 1655 image: centos7-gpdb6-image 1656 file: gpbackup/ci/tasks/scale-tests.yml 1657 on_success: 1658 <<: *ccp_destroy_nvme 1659 on_failure: 1660 *slack_alert 1661 ensure: 1662 <<: *set_failed 1663 1664 - name: scale-5x 1665 plan: 1666 - in_parallel: 1667 - get: centos6-gpdb5-image 1668 - get: gpbackup 1669 passed: [build_gppkgs] 1670 - get: gpdb_binary 1671 resource: bin_gpdb_5x_stable_centos6 1672 - get: ccp_src 1673 - get: gpdb_src 1674 resource: gpdb5_src 1675 - get: scale_schema 1676 {% if nightly_trigger and "gpbackup-release" != pipeline_name %} 1677 - get: nightly-trigger 1678 trigger: true 1679 {% endif %} 1680 - get: gppkgs 1681 passed: [build_gppkgs] 1682 - get: terraform.d 1683 params: 1684 unpack: true 1685 - put: terraform 1686 params: 1687 <<: *ccp_default_params 1688 terraform_source: ccp_src/google-nvme-block-device/ 1689 vars: 1690 instance_type: n1-standard-8 1691 - task: gen_cluster 1692 params: 1693 <<: *ccp_gen_cluster_default_params 1694 file: ccp_src/ci/tasks/gen_cluster.yml 1695 - task: gpinitsystem 1696 file: ccp_src/ci/tasks/gpinitsystem.yml 1697 - task: setup-centos-env 1698 image: centos6-gpdb5-image 1699 file: gpbackup/ci/tasks/setup-centos-env.yml 1700 - task: scale-tests 1701 image: centos6-gpdb5-image 1702 file: gpbackup/ci/tasks/scale-tests.yml 1703 on_success: 1704 <<: *ccp_destroy_nvme 1705 on_failure: 1706 *slack_alert 1707 ensure: 1708 <<: *set_failed 1709 1710 - name: scale-43 1711 plan: 1712 - in_parallel: 1713 - get: centos6-gpdb5-image 1714 - get: gpbackup 1715 passed: [build_gppkgs] 1716 - get: gpdb_binary 1717 resource: bin_gpdb_43_stable 1718 - get: ccp_src 1719 - get: gpdb_src 1720 resource: gpdb5_src 1721 - get: scale_schema 1722 {% if nightly_trigger and "gpbackup-release" != pipeline_name %} 1723 - get: nightly-trigger 1724 trigger: true 1725 {% endif %} 1726 - get: gppkgs 1727 passed: [build_gppkgs] 1728 - get: terraform.d 1729 params: 1730 unpack: true 1731 - put: terraform 1732 params: 1733 <<: *ccp_default_params 1734 terraform_source: ccp_src/google-nvme-block-device/ 1735 vars: 1736 instance_type: n1-standard-8 1737 - task: gen_cluster 1738 params: 1739 <<: *ccp_gen_cluster_default_params 1740 file: ccp_src/ci/tasks/gen_cluster.yml 1741 - task: gpinitsystem 1742 file: ccp_src/ci/tasks/gpinitsystem.yml 1743 - task: setup-centos-env 1744 image: centos6-gpdb5-image 1745 file: gpbackup/ci/tasks/setup-centos-env.yml 1746 - task: scale-tests 1747 image: centos6-gpdb5-image 1748 file: gpbackup/ci/tasks/scale-tests.yml 1749 on_success: 1750 <<: *ccp_destroy_nvme 1751 on_failure: 1752 *slack_alert 1753 ensure: 1754 <<: *set_failed 1755 {% endif %} 1756 1757 {% if "gpbackup-release" == pipeline_name %} 1758 - name: green_gate 1759 plan: 1760 - in_parallel: 1761 - get: github_release_components 1762 passed: [build_gppkgs] 1763 - get: gppkgs 1764 trigger: true 1765 passed: 1766 - build_gppkgs 1767 - GPDB4.3 1768 - GPDB5 1769 - GPDB5-sles11 1770 - GPDB6-ubuntu 1771 - GPDB6 1772 - s3_plugin_tests 1773 - backward-compatibility 1774 - ddboost_plugin_and_boostfs_tests_43 1775 - ddboost_plugin_and_boostfs_tests_5x 1776 - ddboost_plugin_and_boostfs_tests_6x 1777 - gpbackup-manager-tests 1778 - get: gpbackup 1779 passed: [build_gppkgs] 1780 - in_parallel: 1781 - put: gppkgs_rc 1782 params: 1783 file: gppkgs/gpbackup-gppkgs.tar.gz 1784 - put: github_release_components_rc 1785 params: 1786 file: github_release_components/gpbackup-*.tar.gz 1787 1788 - name: push-to-github 1789 plan: 1790 - get: centos6-gpdb5-image 1791 - get: github_release_components_rc 1792 passed: [green_gate] 1793 - task: prepare-for-release 1794 image: centos6-gpdb5-image 1795 config: 1796 platform: linux 1797 inputs: 1798 - name: github_release_components_rc 1799 outputs: 1800 - name: github_release_components_untarred 1801 run: 1802 path: "sh" 1803 args: 1804 - -exc 1805 - | 1806 set -x 1807 tar xvzf github_release_components_rc/*.gz -C github_release_components_untarred 1808 tar xvzf github_release_components_untarred/bin_gpbackup.tar.gz -C github_release_components_untarred 1809 - put: gpbackup_github_release 1810 params: 1811 name: github_release_components_untarred/gpbackup_version 1812 tag: github_release_components_untarred/gpbackup_version 1813 globs: 1814 - github_release_components_untarred/bin/gpbackup 1815 - github_release_components_untarred/bin/gprestore 1816 - github_release_components_untarred/bin/gpbackup_helper 1817 1818 - name: push-to-pivnet 1819 plan: 1820 - in_parallel: 1821 - get: gppkgs 1822 resource: gppkgs_rc 1823 passed: [green_gate] 1824 - get: github_release_components_rc 1825 passed: [green_gate] 1826 - get: gpbackup 1827 passed: [green_gate] 1828 - get: gpbackup-release-license 1829 - get: pivnet_release_cache 1830 - get: centos6-gpdb5-image 1831 - task: update metadata.yml 1832 image: centos6-gpdb5-image 1833 file: gpbackup/ci/tasks/update-metadata.yml 1834 - put: pivnet-upload 1835 params: 1836 metadata_file: workspace/metadata.yml 1837 file_glob: workspace/files-to-upload/* 1838 s3_filepath_prefix: ((dp/$$DEV_PROD$$/pivnet_s3_filepath_prefix)) 1839 - put: pivnet_release_cache 1840 params: 1841 file: workspace/v-* 1842 {% endif %} 1843 1844 {% if "gpbackup-release" != pipeline_name %} 1845 - name: final_gate 1846 plan: 1847 - in_parallel: 1848 - get: gppkgs 1849 trigger: true 1850 passed: 1851 - build_gppkgs 1852 - GPDB4.3 1853 - GPDB5 1854 - GPDB6 1855 - GPDB7 1856 - s3_plugin_tests 1857 - backward-compatibility 1858 - 5X-head-gpbackup-fixed-test 1859 {% if is_prod %} 1860 - GPDB5-sles11 1861 - GPDB6-ubuntu 1862 - ddboost_plugin_and_boostfs_tests_43 1863 - ddboost_plugin_and_boostfs_tests_5x 1864 {% endif %} 1865 - ddboost_plugin_and_boostfs_tests_6x 1866 - gpbackup-manager-tests 1867 {% endif %}