github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/ci/scale/scale_pipeline.yml (about) 1 # USAGE: fly -t dp set-pipeline -p scale_test_refactor -c ~/workspace/gpbackup/ci/scale/scale_pipeline.yml -v gpbackup-git-branch=BRANCH_NAME 2 --- 3 groups: 4 - name: Scale 5 jobs: 6 - build_binaries 7 - build_gppkgs 8 - scale-test 9 10 resource_types: 11 - name: terraform 12 type: registry-image 13 source: 14 repository: ljfranklin/terraform-resource 15 tag: 0.11.14 16 17 - name: slack-notification 18 type: registry-image 19 source: 20 repository: cfcommunity/slack-notification-resource 21 tag: latest 22 23 - name: gcs 24 type: registry-image 25 source: 26 repository: frodenas/gcs-resource 27 28 - name: pivnet 29 type: registry-image 30 source: 31 repository: pivotalcf/pivnet-resource 32 tag: latest-final 33 34 ## ====================================================================== 35 ## _ __ ___ ___ ___ _ _ _ __ ___ ___ ___ 36 ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __| 37 ## | | | __/\__ \ (_) | |_| | | | (_| __/\__ \ 38 ## |_| \___||___/\___/ \__,_|_| \___\___||___/ 39 ## ====================================================================== 40 41 resources: 42 ##### Docker Images ##### 43 - name: centos6-image 44 type: registry-image 45 source: 46 repository: gcr.io/data-gpdb-public-images/gpdb5-centos6-build-test 47 tag: latest 48 49 - name: centos7-image 50 type: registry-image 51 source: 52 repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test 53 tag: latest 54 55 - name: gpdb7-centos7-image 56 type: registry-image 57 source: 58 repository: gcr.io/data-gpdb-public-images/gpdb7-centos7-build 59 tag: latest 60 61 ##### Source Code #### 62 - name: gpbackup 63 type: git 64 icon: github-circle 65 source: 66 uri: https://github.com/cloudberrydb/gpbackup 67 branch: ((gpbackup-git-branch)) 68 69 - name: gpdb6_src 70 type: git 71 icon: github-circle 72 source: 73 uri: https://github.com/greenplum-db/gpdb 74 branch: 6X_STABLE 75 76 - name: gpbackup_manager_src 77 type: git 78 icon: github-circle 79 source: 80 branch: main 81 private_key: ((dp/dev/gp-backup-manager-remote-deploy-key)) 82 uri: ((dp/gp-backup-manager-git-remote)) 83 84 - name: gpbackup_s3_plugin 85 type: git 86 icon: github-circle 87 source: 88 branch: ((dp/dev/gpbackup-s3-plugin-git-branch)) 89 uri: https://github.com/cloudberrydb/gpbackup-s3-plugin 90 91 - name: gpbackup_ddboost_plugin 92 type: git 93 icon: github-circle 94 source: 95 branch: ((dp/gpbackup-ddboost-plugin-branch)) 96 private_key: ((dp/gpbackup-ddboost-plugin-remote-key)) 97 uri: ((dp/gpbackup-ddboost-plugin-git-remote)) 98 99 #### Binaries #### 100 - name: bin_gpdb_6x_stable_centos7 101 type: gcs 102 icon: google 103 source: 104 bucket: ((dp/dev/gcs-bucket)) 105 json_key: ((concourse-gcs-resources-service-account-key)) 106 regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64((dp/dev/rc-build-type-gcs)).tar.gz 107 108 #### Misc Other #### 109 - name: weekly-trigger 110 type: time 111 source: 112 location: America/Los_Angeles 113 interval: 168h # 7 days 114 115 - name: terraform 116 type: terraform 117 source: 118 env: 119 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 120 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 121 GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key)) 122 vars: 123 project_id: ((dp/dev/google-project-id)) 124 storage: 125 access_key_id: ((tf-machine-access-key-id)) 126 secret_access_key: ((tf-machine-secret-access-key)) 127 region_name: ((dp/aws-region)) 128 bucket: ((dp/tf-bucket-name)) 129 bucket_path: clusters-google/ 130 131 - name: slack-alert 132 type: slack-notification 133 source: 134 url: ((dp/webhook_url)) 135 disable: false 136 137 - name: gpbackup-go-components 138 type: s3 139 icon: amazon 140 source: 141 access_key_id: ((aws-bucket-access-key-id)) 142 bucket: ((dp/dev/gpdb-stable-bucket-name)) 143 region_name: ((dp/aws-region)) 144 secret_access_key: ((aws-bucket-secret-access-key)) 145 versioned_file: gpbackup-go-components/go_components.tar.gz 146 147 - name: gppkgs 148 type: s3 149 icon: amazon 150 source: 151 access_key_id: ((aws-bucket-access-key-id)) 152 bucket: ((dp/dev/gpdb-stable-bucket-name)) 153 region_name: ((dp/aws-region)) 154 secret_access_key: ((aws-bucket-secret-access-key)) 155 versioned_file: gppkgs/intermediates/gpbackup-gppkgs.tar.gz 156 157 - name: gpbackup-release-license 158 type: gcs 159 icon: google 160 source: 161 bucket: gpbackup-release-licenses 162 json_key: ((dp/dev/gcp_svc_acct_key)) 163 regexp: open_source_license_VMware_Tanzu_Greenplum_Backup_and_Restore_(.*)_.*.txt 164 165 - name: pivnet_release_cache 166 type: s3 167 icon: amazon 168 source: 169 access_key_id: ((aws-bucket-access-key-id)) 170 bucket: ((dp/dev/pivnet_bucket_name)) 171 region_name: ((dp/aws-region)) 172 secret_access_key: ((aws-bucket-secret-access-key)) 173 regexp: pivnet_release_version/v-(.*) 174 175 - name: ccp_src 176 type: git 177 icon: github-circle 178 source: 179 branch: ((dp/ccp-git-branch)) 180 private_key: ((gp-concourse-cluster-provisioner-git-key)) 181 uri: ((dp/ccp-git-remote)) 182 183 - name: terraform.d 184 type: s3 185 icon: amazon 186 source: 187 access_key_id: ((aws-bucket-access-key-id)) 188 bucket: ccp-terraform-provider-plugins 189 region_name: ((dp/aws-region)) 190 secret_access_key: ((aws-bucket-secret-access-key)) 191 versioned_file: plugin-cache-all.tgz 192 193 ## ====================================================================== 194 ## _ 195 ## __ _ _ __ ___| |__ ___ _ __ ___ 196 ## / _` | '_ \ / __| '_ \ / _ \| '__/ __| 197 ## | (_| | | | | (__| | | | (_) | | \__ \ 198 ## \__,_|_| |_|\___|_| |_|\___/|_| |___/ 199 ## ====================================================================== 200 201 anchors: 202 - &ccp_default_params 203 action: create 204 delete_on_failure: true 205 generate_random_name: true 206 plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 207 terraform_source: ccp_src/google/ 208 209 - &ccp_gen_cluster_default_params 210 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 211 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 212 AWS_DEFAULT_REGION: ((dp/aws-region)) 213 BUCKET_PATH: clusters-google/ 214 BUCKET_NAME: ((dp/tf-bucket-name)) 215 CLOUD_PROVIDER: google 216 217 - &destroy_params 218 action: destroy 219 plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 220 terraform_source: ccp_src/google/ 221 env_name_file: terraform/name 222 vars: 223 aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. 224 aws_ebs_volume_type: standard 225 226 - &ccp_destroy 227 put: terraform 228 params: 229 <<: *destroy_params 230 get_params: 231 action: destroy 232 233 - &ccp_destroy_nvme 234 put: terraform 235 params: 236 <<: *destroy_params 237 terraform_source: ccp_src/google-nvme-block-device/ 238 get_params: 239 action: destroy 240 241 - &set_failed 242 do: 243 - task: on_failure_set_failed 244 config: 245 platform: linux 246 image_resource: 247 type: registry-image 248 source: 249 repository: gcr.io/data-gpdb-public-images/ccp 250 tag: "latest" 251 inputs: 252 - name: ccp_src 253 - name: terraform 254 run: 255 path: 'ccp_src/google/ccp_failed_test.sh' 256 params: 257 GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key)) 258 GOOGLE_PROJECT_ID: ((dp/dev/google-service-account)) 259 GOOGLE_ZONE: ((dp/dev/google-zone)) 260 GOOGLE_SERVICE_ACCOUNT: ((dp/dev/google-service-account)) 261 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 262 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 263 AWS_DEFAULT_REGION: ((dp/tf-machine-region)) 264 BUCKET_PATH: clusters-google/ 265 BUCKET_NAME: ((dp/tf-bucket-name)) 266 267 - &terraform_cluster 268 type: terraform 269 source: 270 env: 271 AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) 272 AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) 273 GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key)) 274 vars: 275 project_id: ((dp/dev/google-service-account)) 276 storage: 277 access_key_id: ((tf-machine-access-key-id)) 278 secret_access_key: ((tf-machine-secret-access-key)) 279 region_name: ((dp/aws-region)) 280 # This is not parameterized, on purpose. All tfstates will go to this spot, 281 # and different teams will place there clusters' tfstate files under different paths 282 bucket: gpdb5-pipeline-dynamic-terraform 283 bucket_path: clusters-google/ 284 285 286 - &slack_alert 287 put: slack-alert 288 params: 289 text: | 290 [gpbackup/$BUILD_JOB_NAME] failed: 291 https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME 292 293 ## ====================================================================== 294 ## _ _ 295 ## (_) ___ | |__ ___ 296 ## | |/ _ \| '_ \/ __| 297 ## | | (_) | |_) \__ \ 298 ## _/ |\___/|_.__/|___/ 299 ## |__/ 300 ## ====================================================================== 301 302 jobs: 303 - name: build_binaries 304 plan: 305 - in_parallel: 306 - get: gpbackup_s3_plugin 307 trigger: true 308 - get: gpbackup_manager_src 309 trigger: true 310 - get: gpbackup 311 trigger: true 312 - get: pivnet_release_cache 313 - task: build-go-binaries 314 file: gpbackup/ci/tasks/build-go-binaries.yml 315 - put: gpbackup-go-components 316 params: 317 file: go_components/go_components.tar.gz 318 319 - name: build_gppkgs 320 plan: 321 - in_parallel: 322 - get: centos7-image 323 - get: gpdb_src 324 resource: gpdb6_src 325 - get: gpbackup-go-components 326 trigger: true 327 passed: [build_binaries] 328 - get: bin_gpdb_6x_stable_centos7 329 - get: gpbackup 330 passed: [build_binaries] 331 - get: gpbackup_ddboost_plugin 332 trigger: true 333 - get: pivnet_release_cache 334 - get: gpbackup-release-license 335 - task: gpbackup-tools-versions 336 image: centos7-image 337 file: gpbackup/ci/tasks/gpbackup-tools-versions.yml 338 - in_parallel: 339 - do: # RHEL 340 - task: build-ddboost-RHEL 341 image: centos7-image 342 file: gpbackup/ci/tasks/build-ddboost.yml 343 input_mapping: 344 bin_gpdb: bin_gpdb_6x_stable_centos7 345 - task: tar-binaries-RHEL 346 image: centos7-image 347 file: gpbackup/ci/tasks/build-os-tars.yml 348 - task: build_gppkgs-RHEL 349 image: centos7-image 350 file: gpbackup/ci/tasks/build-gppkg.yml 351 input_mapping: 352 bin_gpdb: bin_gpdb_6x_stable_centos7 353 output_mapping: 354 gppkgs: rhel-gppkg 355 params: 356 OS: RHEL 357 - task: tar-gppkgs 358 image: centos7-image 359 file: gpbackup/ci/tasks/tar-gppkgs.yml 360 - put: gppkgs 361 params: 362 file: gppkgs/gpbackup-gppkgs.tar.gz 363 364 - name: scale-test 365 plan: 366 - in_parallel: 367 - get: weekly-trigger 368 trigger: true 369 - get: centos6-image 370 - get: centos7-image 371 - get: gpdb7-centos7-image 372 - get: gpbackup 373 passed: [build_gppkgs] 374 - get: gpdb_binary 375 resource: bin_gpdb_6x_stable_centos7 376 - get: ccp_src 377 - get: gpdb_src 378 resource: gpdb6_src 379 - get: gppkgs 380 passed: [build_gppkgs] 381 - get: terraform.d 382 params: 383 unpack: true 384 - put: terraform 385 params: 386 <<: *ccp_default_params 387 terraform_source: ccp_src/google-nvme-block-device/ 388 vars: 389 PLATFORM: centos7 390 instance_type: n2-standard-8 391 disk_size: 1000 392 number_of_nodes: 4 393 segments_per_host: 4 394 - task: gen_cluster 395 params: 396 <<: *ccp_gen_cluster_default_params 397 PLATFORM: centos7 398 file: ccp_src/ci/tasks/gen_cluster.yml 399 - task: gpinitsystem 400 file: ccp_src/ci/tasks/gpinitsystem.yml 401 - task: setup-centos-env 402 image: centos6-image 403 file: gpbackup/ci/tasks/setup-centos-env.yml 404 - task: scale-perf-tests 405 image: centos6-image 406 file: gpbackup/ci/tasks/scale-perf-tests.yml 407 params: 408 RESULTS_LOG_FILE: /tmp/gpbackup.log 409 RESULTS_DATABASE_HOST: 10.122.32.4 410 RESULTS_DATABASE_USER: postgres 411 RESULTS_DATABASE_NAME: gpbackup_refdb 412 RESULTS_DATABASE_PASSWORD: ((dp/dev/gcp_refdb_admin_password)) 413 GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key)) 414 GOOGLE_PROJECT_ID: ((dp/dev/google-project-id)) 415 GOOGLE_ZONE: ((dp/dev/google-zone)) 416 GOOGLE_SERVICE_ACCOUNT: ((dp/dev/google-service-account)) 417 BUCKET_NAME: ((dp/dev/scale_test_bucket)) 418 on_failure: 419 *slack_alert 420 - task: scale-test-slack-notify 421 image: gpdb7-centos7-image 422 file: gpbackup/ci/tasks/scale-tests-slack-notify.yml 423 params: 424 RESULTS_DATABASE_HOST: 10.122.32.4 425 RESULTS_DATABASE_USER: postgres 426 RESULTS_DATABASE_NAME: gpbackup_refdb 427 SLACK_WEBHOOK_URL: ((dp/webhook_url)) 428 RESULTS_DATABASE_PASSWORD: ((dp/dev/gcp_refdb_admin_password))