github.com/greenplum-db/gpbackup@v0.0.0-20240517212602-89daab1885b3/ci/regression/regression_pipeline.yml (about)

     1  # USAGE: fly -t dp set-pipeline  -p regression -c ~/workspace/gpbackup/ci/regression/regression_pipeline.yml -v gpbackup-git-branch=BRANCH_NAME
     2  ---
     3  groups:
     4  - name: all
     5    jobs: 
     6    - build_binaries
     7    - build_gppkgs
     8    - regdb-gpdb6
     9    - regdb-gpdb7
    10    - regdb-gpdb6-to-gpdb7-backup
    11    - regdb-gpdb6-to-gpdb7-restore
    12  
    13  - name: regression
    14    jobs:
    15    - build_binaries
    16    - build_gppkgs
    17    - regdb-gpdb6
    18    - regdb-gpdb7
    19  
    20  - name: migration
    21    jobs: 
    22    - regdb-gpdb6-to-gpdb7-backup
    23    - regdb-gpdb6-to-gpdb7-restore
    24  
    25  resource_types:
    26  - name: terraform
    27    type: registry-image
    28    source:
    29      repository: ljfranklin/terraform-resource
    30      tag: 0.11.14
    31  
    32  - name: slack-notification
    33    type: registry-image
    34    source:
    35      repository: cfcommunity/slack-notification-resource
    36      tag: latest
    37  
    38  - name: gcs
    39    type: registry-image
    40    source:
    41      repository: frodenas/gcs-resource
    42  
    43  - name: pivnet
    44    type: registry-image
    45    source:
    46      repository: pivotalcf/pivnet-resource
    47      tag: latest-final
    48  
    49  ## ======================================================================
    50  ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  ___
    51  ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __|
    52  ## | | |  __/\__ \ (_) | |_| | | | (_|  __/\__ \
    53  ## |_|  \___||___/\___/ \__,_|_|  \___\___||___/
    54  ## ======================================================================
    55  
    56  resources:
    57  ##### Docker Images #####
    58  - name: rocky8-gpdb6-image
    59    type: registry-image
    60    source:
    61      repository: gcr.io/data-gpdb-public-images/gpdb6-rocky8-test
    62      tag: latest
    63  
    64  - name: rocky8-gpdb7-image
    65    type: registry-image
    66    source:
    67      repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test
    68      tag: latest
    69  
    70  ##### Source Code ####
    71  - name: gpbackup
    72    type: git
    73    icon: github-circle
    74    source:
    75      uri: https://github.com/greenplum-db/gpbackup
    76      branch: ((gpbackup-git-branch))
    77  
    78  - name: gpbackup_s3_plugin
    79    type: git
    80    icon: github-circle
    81    source:
    82      branch: ((dp/prod/gpbackup-s3-plugin-git-branch))
    83      uri: https://github.com/greenplum-db/gpbackup-s3-plugin
    84  
    85  - name: gpbackup_ddboost_plugin
    86    type: git
    87    icon: github-circle
    88    source:
    89      branch: ((dp/gpbackup-ddboost-plugin-branch))
    90      private_key: ((dp/gpbackup-ddboost-plugin-remote-key))
    91      uri: ((dp/gpbackup-ddboost-plugin-git-remote))
    92  
    93  - name: gpbackup_manager_src
    94    type: git
    95    icon: github-circle
    96    source:
    97      branch: main
    98      private_key: ((dp/prod/gp-backup-manager-remote-deploy-key))
    99      uri: ((dp/gp-backup-manager-git-remote))
   100  
   101  - name: diffdb_src
   102    type: git
   103    icon: github-circle
   104    source:
   105      uri: https://github.com/AJR-VMware/diffdb
   106      branch: main
   107  
   108  - name: gpdb6_src
   109    type: git
   110    icon: github-circle
   111    source:
   112      uri: https://github.com/greenplum-db/gpdb
   113      branch: 6X_STABLE
   114  
   115  - name: gpdb_main_src
   116    type: git
   117    icon: github-circle
   118    source:
   119      uri: https://github.com/greenplum-db/gpdb
   120      branch: main
   121  
   122  #### Binaries ####
   123  - name: bin_gpdb_6x_rhel8
   124    type: gcs
   125    source:
   126      bucket: ((dp/prod/gcs-ci-bucket))
   127      json_key: ((concourse-gcs-resources-service-account-key))
   128      regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.tar.gz
   129  
   130  - name: bin_gpdb_7x_rhel8
   131    type: gcs
   132    icon: google
   133    source:
   134      bucket: ((dp/prod/gcs-ci-bucket))
   135      json_key: ((concourse-gcs-resources-service-account-key))
   136      regexp: server/published/main/server-rc-(.*)-el8_x86_64.tar.gz
   137  
   138  #### Misc Other ####
   139  - name: weekly-trigger
   140    type: time
   141    source:
   142      location: America/Los_Angeles
   143      start: 8:00 AM
   144      stop: 9:00 AM
   145      days: [Tuesday]
   146  
   147  - name: gp-pkg
   148    type: gcs
   149    icon: google
   150    source:
   151      json_key: ((concourse-gcs-resources-service-account-key))
   152      bucket: pivotal-gpdb-concourse-resources-prod
   153      regexp: gppkg/released/gpdb7/gppkg-(.*).tar.gz
   154  
   155  - name: gpbackup-go-components-rhel8
   156    type: gcs
   157    icon: google
   158    source:
   159      bucket: ((dp/dev/gcs-ci-bucket))
   160      json_key: ((dp/dev/gcp_svc_acct_key))
   161      versioned_file: gpbackup/intermediates/rhel8/go_components.tar.gz
   162  
   163  - name: pivnet_release_cache
   164    type: s3
   165    icon: amazon
   166    source:
   167      access_key_id: ((aws-bucket-access-key-id))
   168      bucket: ((dp/prod/pivnet_bucket_name))
   169      region_name: ((dp/aws-region))
   170      secret_access_key: ((aws-bucket-secret-access-key))
   171      regexp: pivnet_release_version/v-(.*)
   172  
   173  - name: gpbackup-release-license
   174    type: gcs
   175    icon: google
   176    source:
   177      bucket: gpbackup-release-licenses
   178      json_key: ((dp/dev/gcp_svc_acct_key))
   179      regexp: open_source_license_VMware_Greenplum_Backup_and_Restore_(.*)_.*.txt
   180  
   181  - name: icw_dump_gpdb6
   182    type: gcs
   183    source:
   184      bucket: pivotal-gpdb-concourse-resources-intermediates-prod
   185      json_key: ((concourse-gcs-resources-service-account-key))
   186      versioned_file: 6X_STABLE_without_asserts/icw_planner_rocky8_dump/dump.sql.xz
   187  
   188  - name: icw_dump_gpdb7
   189    type: gcs
   190    source:
   191      bucket: pivotal-gpdb-concourse-resources-intermediates-prod
   192      json_key: ((concourse-gcs-resources-service-account-key))
   193      versioned_file: gpdb_main/icw_planner_rhel8_dump/dump.sql.xz
   194  
   195  - name: gpdb6-migration-backup
   196    type: gcs
   197    icon: google
   198    source:
   199      bucket: ((dp/dev/gcs-ci-bucket))
   200      json_key: ((dp/dev/gcp_svc_acct_key))
   201      versioned_file: gpbackup/intermediates/migration-backups/gpdb6/migration-backup.tar.gz
   202  
   203  - name: terraform
   204    type: terraform
   205    source:
   206      env:
   207        AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   208        AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   209        GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
   210      vars:
   211        project_id: ((dp/dev/google-project-id))
   212      storage:
   213        access_key_id: ((tf-machine-access-key-id))
   214        secret_access_key: ((tf-machine-secret-access-key))
   215        region_name: ((dp/aws-region))
   216        bucket: ((dp/tf-bucket-name))
   217        bucket_path: clusters-google/
   218  
   219  - name: slack-alert
   220    type: slack-notification
   221    source:
   222      url: ((dp/webhook_url))
   223      disable: false
   224  
   225  - name: gppkgs
   226    type: gcs
   227    icon: google
   228    source:
   229      bucket: ((dp/dev/gcs-ci-bucket))
   230      json_key: ((dp/dev/gcp_svc_acct_key))
   231      versioned_file: gpbackup/intermediates/gpbackup-gppkgs.tar.gz
   232  
   233  - name: ccp_src
   234    type: git
   235    icon: github-circle
   236    source:
   237      branch: ((dp/ccp-git-branch))
   238      private_key: ((gp-concourse-cluster-provisioner-git-key))
   239      uri: ((dp/ccp-git-remote))
   240  
   241  - name: terraform.d
   242    type: s3
   243    icon: amazon
   244    source:
   245      access_key_id: ((aws-bucket-access-key-id))
   246      bucket: ccp-terraform-provider-plugins
   247      region_name: ((dp/aws-region))
   248      secret_access_key: ((aws-bucket-secret-access-key))
   249      versioned_file: plugin-cache-all.tgz
   250  
   251  ## ======================================================================
   252  ##                   _
   253  ##   __ _ _ __   ___| |__   ___  _ __ ___
   254  ##  / _` | '_ \ / __| '_ \ / _ \| '__/ __|
   255  ## | (_| | | | | (__| | | | (_) | |  \__ \
   256  ##  \__,_|_| |_|\___|_| |_|\___/|_|  |___/
   257  ## ======================================================================
   258  
   259  anchors:
   260  - &ccp_default_params
   261    action: create
   262    delete_on_failure: true
   263    generate_random_name: true
   264    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   265    terraform_source: ccp_src/google-nvme-block-device/
   266  
   267  - &ccp_gen_cluster_default_params
   268    AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   269    AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   270    AWS_DEFAULT_REGION: ((dp/aws-region))
   271    BUCKET_PATH: clusters-google/
   272    BUCKET_NAME: ((dp/tf-bucket-name))
   273    CLOUD_PROVIDER: google
   274  
   275  - &destroy_params
   276    action: destroy
   277    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   278    terraform_source: ccp_src/google-nvme-block-device/
   279    env_name_file: terraform/name
   280    vars:
   281      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
   282      aws_ebs_volume_type: standard
   283  
   284  - &ccp_destroy_nvme
   285    put: terraform
   286    params:
   287      <<: *destroy_params
   288    get_params:
   289      action: destroy
   290  
   291  - &slack_alert
   292    put: slack-alert
   293    params:
   294      text: |
   295        [regression/$BUILD_JOB_NAME] failed:
   296        https://dp.ci.gpdb.pivotal.io/teams/main/pipelines/regression/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME
   297  
   298  ## ======================================================================
   299  ##    _       _
   300  ##   (_) ___ | |__  ___
   301  ##   | |/ _ \| '_ \/ __|
   302  ##   | | (_) | |_) \__ \
   303  ##  _/ |\___/|_.__/|___/
   304  ## |__/
   305  ## ======================================================================
   306  
   307  jobs:
   308  - name: build_binaries
   309    plan:
   310    - in_parallel:
   311      - get: gpbackup_s3_plugin
   312        trigger: true
   313      - get: gpbackup_manager_src
   314        trigger: true
   315      - get: gpbackup
   316        trigger: true
   317      - get: pivnet_release_cache
   318      - get: rocky8-gpdb7-image
   319    - task: build-go-binaries-rhel8
   320      file: gpbackup/ci/tasks/build-go-binaries.yml
   321      image: rocky8-gpdb7-image
   322      params:
   323        OS: RHEL8
   324    - put: gpbackup-go-components-rhel8
   325      params:
   326        file: go_components/go_components.tar.gz
   327  
   328  - name: build_gppkgs
   329    plan:
   330    - in_parallel:
   331      - get: rocky8-gpdb6-image
   332      - get: gp-pkg
   333      - get: gpdb_src
   334        resource: gpdb6_src
   335      - get: gpbackup-go-components-rhel8
   336        trigger: true
   337        passed: [build_binaries]
   338      - get: bin_gpdb_6x_rhel8
   339      - get: gpbackup
   340        passed: [build_binaries]
   341      - get: gpbackup_ddboost_plugin
   342      - get: pivnet_release_cache
   343      - get: gpbackup-release-license
   344    - task: gpbackup-tools-versions
   345      image: rocky8-gpdb6-image
   346      file: gpbackup/ci/tasks/gpbackup-tools-versions.yml
   347    - in_parallel:
   348      - do: # RHEL8
   349        - task: build-ddboost-rhel8
   350          image: rocky8-gpdb6-image
   351          file: gpbackup/ci/tasks/build-ddboost.yml
   352          input_mapping:
   353            bin_gpdb: bin_gpdb_6x_rhel8
   354        - task: tar-binaries-rhel8
   355          image: rocky8-gpdb6-image
   356          file: gpbackup/ci/tasks/build-os-tars.yml
   357          input_mapping:
   358            gpbackup-go-components: gpbackup-go-components-rhel8
   359          output_mapping:
   360            gpbackup_tar: gpbackup_tar_rhel8
   361        - task: build_gppkgs-rhel8
   362          image: rocky8-gpdb6-image
   363          file: gpbackup/ci/tasks/build-gppkg.yml
   364          input_mapping:
   365            bin_gpdb: bin_gpdb_6x_rhel8
   366            gpbackup_tar: gpbackup_tar_rhel8
   367          output_mapping:
   368            gppkgs: rhel8-gppkg
   369          params:
   370            OS: RHEL8
   371    - task: tar-gppkgs
   372      image: rocky8-gpdb6-image
   373      config:
   374        platform: linux
   375        inputs:
   376          - name: rhel8-gppkg
   377        outputs:
   378          - name: gppkgs
   379        run:
   380          path: "sh"
   381          args:
   382            - -exc
   383            - |
   384              set -ex
   385              mv rhel8-gppkg/* gppkgs/
   386              pushd gppkgs
   387                  tar cvzf gpbackup-gppkgs.tar.gz *
   388              popd
   389    - put: gppkgs
   390      params:
   391        file: gppkgs/gpbackup-gppkgs.tar.gz
   392  
   393  - name: regdb-gpdb6
   394    plan:
   395    - in_parallel:
   396      - get: weekly-trigger
   397        trigger: true
   398      - get: rocky8-gpdb6-image
   399      - get: gpbackup
   400        trigger: true
   401        passed: [build_gppkgs]
   402      - get: diffdb_src
   403      - get: gpdb_binary
   404        resource: bin_gpdb_6x_rhel8
   405      - get: ccp_src
   406      - get: gpdb_src
   407        resource: gpdb6_src
   408      - get: gppkgs
   409      - get: icw_dump
   410        resource: icw_dump_gpdb6
   411      - get: terraform.d
   412        params:
   413          unpack: true
   414    - put: terraform
   415      params:
   416        <<: *ccp_default_params
   417        vars:
   418          instance_type: n1-standard-8
   419          PLATFORM: rocky8
   420    - task: gen_cluster
   421      params:
   422        <<: *ccp_gen_cluster_default_params
   423        PLATFORM: rocky8
   424      file: ccp_src/ci/tasks/gen_cluster.yml
   425    - task: gpinitsystem
   426      file: ccp_src/ci/tasks/gpinitsystem.yml
   427    - task: setup-cluster-env
   428      image: rocky8-gpdb6-image
   429      file: gpbackup/ci/tasks/setup-cluster-env.yml
   430    - task: icw-roundtrip
   431      image: rocky8-gpdb6-image
   432      file: gpbackup/ci/tasks/icw-roundtrip.yml
   433      params:
   434        OS: RHEL8
   435    on_success:
   436      <<: *ccp_destroy_nvme
   437    on_failure:
   438      *slack_alert
   439  
   440  - name: regdb-gpdb7
   441    plan:
   442    - in_parallel:
   443      - get: weekly-trigger
   444        trigger: true
   445      - get: rocky8-gpdb7-image
   446      - get: gp-pkg
   447      - get: gpbackup
   448        trigger: true
   449        passed: [build_gppkgs]
   450      - get: gpdb_binary
   451        resource: bin_gpdb_7x_rhel8
   452      - get: ccp_src
   453      - get: gpdb_src
   454        resource: gpdb_main_src
   455      - get: gppkgs
   456      - get: diffdb_src
   457      - get: icw_dump
   458        resource: icw_dump_gpdb7
   459      - get: terraform.d
   460        params:
   461          unpack: true
   462    - put: terraform
   463      params:
   464        <<: *ccp_default_params
   465        vars:
   466          instance_type: n1-standard-8
   467          PLATFORM: rocky8-gpdb7
   468    - task: gen_cluster
   469      params:
   470        <<: *ccp_gen_cluster_default_params
   471        PLATFORM: rocky8-gpdb7
   472      file: ccp_src/ci/tasks/gen_cluster.yml
   473    - task: gpinitsystem
   474      file: ccp_src/ci/tasks/gpinitsystem.yml
   475    - task: setup-cluster-env
   476      image: rocky8-gpdb7-image
   477      file: gpbackup/ci/tasks/setup-cluster-env.yml
   478    - task: icw-roundtrip
   479      image: rocky8-gpdb7-image
   480      file: gpbackup/ci/tasks/icw-roundtrip.yml
   481      params:
   482        OS: RHEL8
   483    on_success:
   484      <<: *ccp_destroy_nvme
   485    on_failure:
   486      *slack_alert
   487  
   488  - name: regdb-gpdb6-to-gpdb7-backup
   489    plan:
   490    - in_parallel:
   491      - get: weekly-trigger
   492        trigger: true
   493      - get: rocky8-gpdb6-image
   494      - get: gpbackup
   495        trigger: true
   496        passed: [build_gppkgs]
   497      - get: bin_gpdb_6x_rhel8
   498        resource: 
   499      - get: gpdb_src
   500        resource: gpdb6_src
   501      - get: gppkgs
   502      - get: icw_dump
   503        resource: icw_dump_gpdb6
   504    - task: icw-migr-backup
   505      image: rocky8-gpdb6-image
   506      file: gpbackup/ci/tasks/icw-migr-backup.yml
   507      input_mapping:
   508        bin_gpdb: bin_gpdb_6x_rhel8
   509    - put: gpdb6-migration-backup
   510      params:
   511          file: migration-artifacts/migration-backup.tar.gz
   512    on_failure:
   513      *slack_alert
   514  
   515  - name: regdb-gpdb6-to-gpdb7-restore
   516    plan:
   517    - in_parallel:
   518      - get: rocky8-gpdb7-image
   519      - get: gp-pkg
   520      - get: gpbackup
   521      - get: bin_gpdb_7x_rhel8
   522        resource: 
   523      - get: gpdb_src
   524        resource: gpdb_main_src
   525      - get: gppkgs
   526      - get: migration-backup
   527        trigger: true
   528        resource: gpdb6-migration-backup 
   529        passed: [regdb-gpdb6-to-gpdb7-backup]
   530    - task: icw-migr-restore
   531      image: rocky8-gpdb7-image
   532      file: gpbackup/ci/tasks/icw-migr-restore.yml
   533      input_mapping:
   534        bin_gpdb: bin_gpdb_7x_rhel8
   535    on_failure:
   536      *slack_alert