github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/ci/regression/regression_pipeline.yml (about)

     1  # USAGE: fly -t dp set-pipeline  -p regression -c ~/workspace/gpbackup/ci/regression/regression_pipeline.yml -v gpbackup-git-branch=BRANCH_NAME
     2  ---
     3  groups:
     4  - name: Regression
     5    jobs:
     6    - build_binaries
     7    - build_gppkgs
     8    - regdb-GPDB6
     9    - regdb-GPDB7
    10  
    11  resource_types:
    12  - name: terraform
    13    type: registry-image
    14    source:
    15      repository: ljfranklin/terraform-resource
    16      tag: 0.11.14
    17  
    18  - name: slack-notification
    19    type: registry-image
    20    source:
    21      repository: cfcommunity/slack-notification-resource
    22      tag: latest
    23  
    24  - name: gcs
    25    type: registry-image
    26    source:
    27      repository: frodenas/gcs-resource
    28  
    29  - name: pivnet
    30    type: registry-image
    31    source:
    32      repository: pivotalcf/pivnet-resource
    33      tag: latest-final
    34  
    35  ## ======================================================================
    36  ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  ___
    37  ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __|
    38  ## | | |  __/\__ \ (_) | |_| | | | (_|  __/\__ \
    39  ## |_|  \___||___/\___/ \__,_|_|  \___\___||___/
    40  ## ======================================================================
    41  
    42  resources:
    43  ##### Docker Images #####  
    44  - name: centos7-image
    45    type: registry-image
    46    source:
    47      repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test
    48      tag: latest
    49  
    50  - name: rocky8-gpdb7-image
    51    type: registry-image
    52    source:
    53      repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test
    54      tag: latest
    55  
    56  ##### Source Code ####
    57  - name: gpbackup
    58    type: git
    59    icon: github-circle
    60    source:
    61      uri: https://github.com/cloudberrydb/gpbackup
    62      branch: ((gpbackup-git-branch))
    63  
    64  - name: diffdb_src
    65    type: git
    66    icon: github-circle
    67    source:
    68      uri: https://github.com/AJR-VMware/diffdb
    69      branch: main
    70  
    71  - name: gpdb6_src
    72    type: git
    73    icon: github-circle
    74    source:
    75      uri: https://github.com/greenplum-db/gpdb
    76      branch: 6X_STABLE
    77  
    78  - name: gpdb_main_src
    79    type: git
    80    icon: github-circle
    81    source:
    82      uri: https://github.com/greenplum-db/gpdb
    83      branch: main
    84  
    85  - name: gpbackup_manager_src
    86    type: git
    87    icon: github-circle
    88    source:
    89      branch: main
    90      private_key: ((dp/dev/gp-backup-manager-remote-deploy-key))
    91      uri: ((dp/gp-backup-manager-git-remote))
    92  
    93  - name: gpbackup_s3_plugin
    94    type: git
    95    icon: github-circle
    96    source:
    97      branch: ((dp/dev/gpbackup-s3-plugin-git-branch))
    98      uri: https://github.com/cloudberrydb/gpbackup-s3-plugin
    99  
   100  - name: gpbackup_ddboost_plugin
   101    type: git
   102    icon: github-circle
   103    source:
   104      branch: ((dp/gpbackup-ddboost-plugin-branch)) 
   105      private_key: ((dp/gpbackup-ddboost-plugin-remote-key))
   106      uri: ((dp/gpbackup-ddboost-plugin-git-remote))
   107  
   108  #### Binaries ####
   109  - name: bin_gpdb_6x_stable_centos7
   110    type: gcs
   111    icon: google
   112    source:
   113      bucket: ((dp/prod/gcs-bucket))
   114      json_key: ((concourse-gcs-resources-service-account-key))
   115      regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64((dp/dev/rc-build-type-gcs)).tar.gz
   116  
   117  - name: bin_gpdb_rhel8
   118    type: gcs
   119    icon: google
   120    source:
   121      bucket: ((dp/prod/gcs-bucket))
   122      json_key: ((concourse-gcs-resources-service-account-key))
   123      regexp: server/published/main/server-rc-(.*)-rhel8_x86_64.tar.gz
   124      
   125  #### Misc Other ####
   126  - name: gpbackup-release-license
   127    type: gcs
   128    icon: google
   129    source:
   130      bucket: gpbackup-release-licenses
   131      json_key: ((dp/dev/gcp_svc_acct_key))
   132      regexp: open_source_license_VMware_Tanzu_Greenplum_Backup_and_Restore_(.*)_.*.txt
   133  
   134  - name: icw_dump_GPDB6
   135    type: gcs
   136    source:
   137      bucket: pivotal-gpdb-concourse-resources-intermediates-prod
   138      json_key: ((concourse-gcs-resources-service-account-key))
   139      versioned_file: 6X_STABLE_without_asserts/icw_planner_centos6_dump/dump.sql.xz
   140  
   141  - name: icw_dump_GPDB7
   142    type: gcs
   143    source:
   144      bucket: pivotal-gpdb-concourse-resources-intermediates-prod
   145      json_key: ((concourse-gcs-resources-service-account-key))
   146      versioned_file: gpdb_main/icw_planner_rhel8_dump/dump.sql.xz
   147  
   148  - name: pivnet_release_cache
   149    type: s3
   150    icon: amazon
   151    source:
   152      access_key_id: ((aws-bucket-access-key-id))
   153      bucket: ((dp/dev/pivnet_bucket_name))
   154      region_name: ((dp/aws-region))
   155      secret_access_key: ((aws-bucket-secret-access-key))
   156      regexp: pivnet_release_version/v-(.*)
   157      
   158  - name: terraform
   159    type: terraform
   160    source:
   161      env:
   162        AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   163        AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   164        GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
   165      vars:
   166        project_id: ((dp/dev/google-project-id))
   167      storage:
   168        access_key_id: ((tf-machine-access-key-id))
   169        secret_access_key: ((tf-machine-secret-access-key))
   170        region_name: ((dp/aws-region))
   171        bucket: ((dp/tf-bucket-name))
   172        bucket_path: clusters-google/
   173  
   174  - name: slack-alert
   175    type: slack-notification
   176    source:
   177      url: ((dp/webhook_url))
   178      disable: false
   179  
   180  - name: gpbackup-go-components
   181    type: s3
   182    icon: amazon
   183    source:
   184      access_key_id: ((aws-bucket-access-key-id))
   185      bucket: ((dp/dev/gpdb-stable-bucket-name))
   186      region_name: ((dp/aws-region))
   187      secret_access_key: ((aws-bucket-secret-access-key))
   188      versioned_file: gpbackup-go-components/go_components.tar.gz
   189  
   190  - name: gppkgs
   191    type: s3
   192    icon: amazon
   193    source:
   194      access_key_id: ((aws-bucket-access-key-id))
   195      bucket: ((dp/dev/gpdb-stable-bucket-name))
   196      region_name: ((dp/aws-region))
   197      secret_access_key: ((aws-bucket-secret-access-key))
   198      versioned_file: gppkgs/intermediates/gpbackup-gppkgs.tar.gz
   199  
   200  - name: ccp_src
   201    type: git
   202    icon: github-circle
   203    source:
   204      branch: ((dp/ccp-git-branch))
   205      private_key: ((gp-concourse-cluster-provisioner-git-key))
   206      uri: ((dp/ccp-git-remote))
   207  
   208  - name: terraform.d
   209    type: s3
   210    icon: amazon
   211    source:
   212      access_key_id: ((aws-bucket-access-key-id))
   213      bucket: ccp-terraform-provider-plugins
   214      region_name: ((dp/aws-region))
   215      secret_access_key: ((aws-bucket-secret-access-key))
   216      versioned_file: plugin-cache-all.tgz
   217  
   218  ## ======================================================================
   219  ##                   _
   220  ##   __ _ _ __   ___| |__   ___  _ __ ___
   221  ##  / _` | '_ \ / __| '_ \ / _ \| '__/ __|
   222  ## | (_| | | | | (__| | | | (_) | |  \__ \
   223  ##  \__,_|_| |_|\___|_| |_|\___/|_|  |___/
   224  ## ======================================================================
   225  
   226  anchors:
   227  - &ccp_default_params
   228    action: create
   229    delete_on_failure: true
   230    generate_random_name: true
   231    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   232    terraform_source: ccp_src/google/
   233  
   234  - &ccp_gen_cluster_default_params
   235    AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   236    AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   237    AWS_DEFAULT_REGION: ((dp/aws-region))
   238    BUCKET_PATH: clusters-google/
   239    BUCKET_NAME: ((dp/tf-bucket-name))
   240    CLOUD_PROVIDER: google
   241  
   242  - &destroy_params
   243    action: destroy
   244    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   245    terraform_source: ccp_src/google/
   246    env_name_file: terraform/name
   247    vars:
   248      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
   249      aws_ebs_volume_type: standard
   250  
   251  - &ccp_destroy
   252    put: terraform
   253    params:
   254      <<: *destroy_params
   255    get_params:
   256      action: destroy
   257  
   258  - &ccp_destroy_nvme
   259    put: terraform
   260    params:
   261      <<: *destroy_params
   262      terraform_source: ccp_src/google-nvme-block-device/
   263    get_params:
   264      action: destroy
   265  
   266  - &terraform_cluster
   267    type: terraform
   268    source:
   269      env:
   270        AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   271        AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   272        GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
   273      vars:
   274        project_id: ((dp/dev/google-service-account))
   275      storage:
   276        access_key_id: ((tf-machine-access-key-id))
   277        secret_access_key: ((tf-machine-secret-access-key))
   278        region_name: ((dp/aws-region))
   279        # This is not parameterized, on purpose. All tfstates will go to this spot,
   280        # and different teams will place there clusters' tfstate files under different paths
   281        bucket: gpdb5-pipeline-dynamic-terraform
   282        bucket_path: clusters-google/
   283  
   284  
   285  - &slack_alert
   286    put: slack-alert
   287    params:
   288      text: |
   289        [gpbackup/$BUILD_JOB_NAME] failed:
   290        https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME
   291  
   292  ## ======================================================================
   293  ##    _       _
   294  ##   (_) ___ | |__  ___
   295  ##   | |/ _ \| '_ \/ __|
   296  ##   | | (_) | |_) \__ \
   297  ##  _/ |\___/|_.__/|___/
   298  ## |__/
   299  ## ======================================================================
   300  
   301  jobs:
   302  - name: build_binaries
   303    plan:
   304    - in_parallel:
   305      - get: gpbackup_s3_plugin
   306        trigger: true
   307      - get: gpbackup_manager_src
   308        trigger: true
   309      - get: gpbackup
   310        trigger: true
   311      - get: pivnet_release_cache
   312    - task: build-go-binaries
   313      file: gpbackup/ci/tasks/build-go-binaries.yml
   314    - put: gpbackup-go-components
   315      params:
   316        file: go_components/go_components.tar.gz
   317    
   318  - name: build_gppkgs
   319    plan:
   320    - in_parallel:
   321      - get: centos7-image
   322      - get: gpdb_src
   323        resource: gpdb6_src
   324      - get: gpbackup-go-components
   325        trigger: true
   326        passed: [build_binaries]
   327      - get: bin_gpdb_6x_stable_centos7
   328      - get: gpbackup
   329        passed: [build_binaries]
   330      - get: gpbackup_ddboost_plugin
   331        trigger: true
   332      - get: pivnet_release_cache
   333      - get: gpbackup-release-license
   334    - task: gpbackup-tools-versions
   335      image: centos7-image
   336      file: gpbackup/ci/tasks/gpbackup-tools-versions.yml
   337    - in_parallel:
   338      - do:
   339        - task: build-ddboost-RHEL
   340          image: centos7-image
   341          file: gpbackup/ci/tasks/build-ddboost.yml
   342          input_mapping:
   343            bin_gpdb: bin_gpdb_6x_stable_centos7
   344        - task: tar-binaries-RHEL
   345          image: centos7-image
   346          file: gpbackup/ci/tasks/build-os-tars.yml
   347        - task: build_gppkgs-RHEL
   348          image: centos7-image
   349          file: gpbackup/ci/tasks/build-gppkg.yml
   350          input_mapping:
   351            bin_gpdb: bin_gpdb_6x_stable_centos7
   352          output_mapping:
   353            gppkgs: rhel-gppkg
   354          params:
   355            OS: RHEL
   356    - task: tar-gppkgs
   357      image: centos7-image
   358      file: gpbackup/ci/tasks/tar-gppkgs.yml
   359    - put: gppkgs
   360      params:
   361        file: gppkgs/gpbackup-gppkgs.tar.gz
   362  
   363  - name: regdb-GPDB6
   364    plan:
   365    - in_parallel:
   366      - get: centos7-image
   367      - get: gpbackup
   368        passed: [build_gppkgs]
   369      - get: diffdb_src
   370      - get: gpdb_binary
   371        resource: bin_gpdb_6x_stable_centos7
   372      - get: ccp_src
   373      - get: gpdb_src
   374        resource: gpdb6_src
   375      - get: gppkgs
   376        trigger: true
   377        passed: [build_gppkgs]
   378      - get: icw_dump
   379        resource: icw_dump_GPDB6
   380      - get: terraform.d
   381        params:
   382          unpack: true
   383    - put: terraform
   384      params:
   385        <<: *ccp_default_params
   386        terraform_source: ccp_src/google-nvme-block-device/
   387        vars:
   388          instance_type: n1-standard-8
   389          PLATFORM: centos7
   390    - task: gen_cluster
   391      params:
   392        <<: *ccp_gen_cluster_default_params
   393        PLATFORM: centos7
   394      file: ccp_src/ci/tasks/gen_cluster.yml
   395    - task: gpinitsystem
   396      file: ccp_src/ci/tasks/gpinitsystem.yml
   397    - task: setup-centos-env
   398      image: centos7-image
   399      file: gpbackup/ci/tasks/setup-centos-env.yml
   400    - task: icw-roundtrip
   401      image: centos7-image
   402      file: gpbackup/ci/tasks/icw-roundtrip.yml
   403    on_success:
   404      <<: *ccp_destroy_nvme
   405    on_failure:
   406      *slack_alert
   407        
   408  - name: regdb-GPDB7
   409    plan:
   410    - in_parallel:
   411      - get: rocky8-gpdb7-image
   412      - get: gpbackup
   413        passed: [build_gppkgs]
   414      - get: gpdb_binary
   415        resource: bin_gpdb_rhel8
   416      - get: ccp_src
   417      - get: gpdb_src
   418        resource: gpdb_main_src
   419      - get: gppkgs
   420        trigger: true
   421        passed: [build_gppkgs]
   422      - get: diffdb_src
   423      - get: icw_dump
   424        resource: icw_dump_GPDB7
   425      - get: terraform.d
   426        params:
   427          unpack: true
   428    - put: terraform
   429      params:
   430        <<: *ccp_default_params
   431        terraform_source: ccp_src/google-nvme-block-device/
   432        vars:
   433          instance_type: n1-standard-8
   434          PLATFORM: rhel8-gpdb7
   435    - task: gen_cluster
   436      params:
   437        <<: *ccp_gen_cluster_default_params
   438        PLATFORM: rhel8-gpdb7
   439      file: ccp_src/ci/tasks/gen_cluster.yml
   440    - task: gpinitsystem
   441      file: ccp_src/ci/tasks/gpinitsystem.yml
   442    - task: setup-centos-env
   443      image: rocky8-gpdb7-image
   444      file: gpbackup/ci/tasks/setup-centos-env.yml
   445      params:
   446        default_ami_user: rhel
   447    - task: icw-roundtrip
   448      image: rocky8-gpdb7-image
   449      file: gpbackup/ci/tasks/icw-roundtrip.yml
   450    on_success:
   451      <<: *ccp_destroy_nvme
   452    on_failure:
   453      *slack_alert