github.com/greenplum-db/gpbackup@v0.0.0-20240517212602-89daab1885b3/ci/scale/scale_pipeline.yml (about)

     1  # USAGE: fly -t dp set-pipeline  -p scale_test_refactor -c ~/workspace/gpbackup/ci/scale/scale_pipeline.yml -v gpbackup-git-branch=BRANCH_NAME
     2  ---
     3  groups:
     4  - name: scale
     5    jobs:
     6    - load-data-gpdb6
     7    - scale-perf-tests-gpdb6
     8  
     9  resource_types:
    10  - name: terraform
    11    type: registry-image
    12    source:
    13      repository: ljfranklin/terraform-resource
    14      tag: 0.11.14
    15  
    16  - name: slack-notification
    17    type: registry-image
    18    source:
    19      repository: cfcommunity/slack-notification-resource
    20      tag: latest
    21  
    22  - name: gcs
    23    type: registry-image
    24    source:
    25      repository: frodenas/gcs-resource
    26  
    27  - name: pivnet
    28    type: registry-image
    29    source:
    30      repository: pivotalcf/pivnet-resource
    31      tag: latest-final
    32  
    33  ## ======================================================================
    34  ##  _ __ ___  ___  ___  _   _ _ __ ___ ___  ___
    35  ## | '__/ _ \/ __|/ _ \| | | | '__/ __/ _ \/ __|
    36  ## | | |  __/\__ \ (_) | |_| | | | (_|  __/\__ \
    37  ## |_|  \___||___/\___/ \__,_|_|  \___\___||___/
    38  ## ======================================================================
    39  
    40  resources:
    41  ##### Docker Images #####
    42  - name: rocky8-gpdb6-image
    43    type: registry-image
    44    source:
    45      repository: gcr.io/data-gpdb-public-images/gpdb6-rocky8-test
    46      tag: latest
    47  
    48  ##### Source Code ####
    49  - name: gpbackup
    50    type: git
    51    icon: github-circle
    52    source:
    53      uri: https://github.com/greenplum-db/gpbackup
    54      branch: ((gpbackup-git-branch))
    55  
    56  - name: gpdb6_src
    57    type: git
    58    icon: github-circle
    59    source:
    60      uri: https://github.com/greenplum-db/gpdb
    61      branch: 6X_STABLE
    62  
    63  - name: ccp_src
    64    type: git
    65    icon: github-circle
    66    source:
    67      branch: ((dp/ccp-git-branch))
    68      private_key: ((gp-concourse-cluster-provisioner-git-key))
    69      uri: ((dp/ccp-git-remote))
    70  
    71  #### Binaries ####
    72  - name: bin_gpdb_6x_rhel8
    73    type: gcs
    74    source:
    75      bucket: ((dp/prod/gcs-ci-bucket))
    76      json_key: ((concourse-gcs-resources-service-account-key))
    77      regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.tar.gz
    78  
    79  #### Misc Other ####
    80  - name: weekly-trigger
    81    type: time
    82    source:
    83      location: America/Los_Angeles
    84      start: 8:00 AM
    85      end: 9:00 AM
    86      days: [Monday]
    87  
    88  - name: terraform
    89    type: terraform
    90    source:
    91      env:
    92        AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
    93        AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
    94        GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
    95      vars:
    96        project_id: ((dp/dev/google-project-id))
    97      storage:
    98        access_key_id: ((tf-machine-access-key-id))
    99        secret_access_key: ((tf-machine-secret-access-key))
   100        region_name: ((dp/aws-region))
   101        bucket: ((dp/tf-bucket-name))
   102        bucket_path: clusters-google/
   103  
   104  - name: slack-alert
   105    type: slack-notification
   106    source:
   107      url: ((dp/webhook_url))
   108      disable: false
   109  
   110  - name: gppkgs
   111    type: gcs
   112    icon: google
   113    source:
   114      bucket: ((dp/dev/gcs-ci-bucket))
   115      json_key: ((dp/dev/gcp_svc_acct_key))
   116      versioned_file: gpbackup/intermediates/gpbackup-gppkgs.tar.gz
   117  
   118  - name: cluster-metadata
   119    type: gcs
   120    source:
   121      bucket: ((dp/dev/gcs-ci-bucket))
   122      json_key: ((dp/dev/gcp_svc_acct_key))
   123      versioned_file: gpbackup/intermediates/cluster-metadata.tar.gz
   124  
   125  - name: terraform.d
   126    type: s3
   127    icon: amazon
   128    source:
   129      access_key_id: ((aws-bucket-access-key-id))
   130      bucket: ccp-terraform-provider-plugins
   131      region_name: ((dp/aws-region))
   132      secret_access_key: ((aws-bucket-secret-access-key))
   133      versioned_file: plugin-cache-all.tgz
   134  
   135  ## ======================================================================
   136  ##                   _
   137  ##   __ _ _ __   ___| |__   ___  _ __ ___
   138  ##  / _` | '_ \ / __| '_ \ / _ \| '__/ __|
   139  ## | (_| | | | | (__| | | | (_) | |  \__ \
   140  ##  \__,_|_| |_|\___|_| |_|\___/|_|  |___/
   141  ## ======================================================================
   142  
   143  anchors:
   144  - &ccp_default_params
   145    action: create
   146    delete_on_failure: true
   147    generate_random_name: true
   148    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   149    terraform_source: ccp_src/google/
   150  
   151  - &ccp_gen_cluster_default_params
   152    AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   153    AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   154    AWS_DEFAULT_REGION: ((dp/aws-region))
   155    BUCKET_PATH: clusters-google/
   156    BUCKET_NAME: ((dp/tf-bucket-name))
   157    CLOUD_PROVIDER: google
   158  
   159  - &destroy_params
   160    action: destroy
   161    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   162    terraform_source: ccp_src/google/
   163    env_name_file: terraform/name
   164    vars:
   165      aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
   166      aws_ebs_volume_type: standard
   167  
   168  - &ccp_destroy
   169    put: terraform
   170    params:
   171      <<: *destroy_params
   172    get_params:
   173      action: destroy
   174  
   175  - &ccp_destroy_nvme
   176    put: terraform
   177    params:
   178      <<: *destroy_params
   179      terraform_source: ccp_src/google-nvme-block-device/
   180    get_params:
   181      action: destroy
   182  
   183  - &set_failed
   184    do:
   185      - task: on_failure_set_failed
   186        config:
   187          platform: linux
   188          image_resource:
   189            type: registry-image
   190            source:
   191              repository: gcr.io/data-gpdb-public-images/ccp
   192              tag: "latest"
   193          inputs:
   194            - name: ccp_src
   195            - name: terraform
   196          run:
   197            path: 'ccp_src/google/ccp_failed_test.sh'
   198          params:
   199            GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
   200            GOOGLE_PROJECT_ID: ((dp/dev/google-service-account))
   201            GOOGLE_ZONE: ((dp/dev/google-zone))
   202            GOOGLE_SERVICE_ACCOUNT: ((dp/dev/google-service-account))
   203            AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   204            AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   205            AWS_DEFAULT_REGION: ((dp/tf-machine-region))
   206            BUCKET_PATH: clusters-google/
   207            BUCKET_NAME: ((dp/tf-bucket-name))
   208  
   209  - &terraform_cluster
   210    type: terraform
   211    source:
   212      env:
   213        AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id))
   214        AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key))
   215        GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
   216      vars:
   217        project_id: ((dp/dev/google-service-account))
   218      storage:
   219        access_key_id: ((tf-machine-access-key-id))
   220        secret_access_key: ((tf-machine-secret-access-key))
   221        region_name: ((dp/aws-region))
   222        # This is not parameterized, on purpose. All tfstates will go to this spot,
   223        # and different teams will place there clusters' tfstate files under different paths
   224        bucket: gpdb5-pipeline-dynamic-terraform
   225        bucket_path: clusters-google/
   226  
   227  
   228  - &slack_alert
   229    put: slack-alert
   230    params:
   231      text: |
   232        [gpbackup/$BUILD_JOB_NAME] failed:
   233        https://dp.ci.gpdb.pivotal.io/teams/main/pipelines/scale_test_refactor/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME
   234  
   235  ## ======================================================================
   236  ##    _       _
   237  ##   (_) ___ | |__  ___
   238  ##   | |/ _ \| '_ \/ __|
   239  ##   | | (_) | |_) \__ \
   240  ##  _/ |\___/|_.__/|___/
   241  ## |__/
   242  ## ======================================================================
   243  
   244  jobs:
   245  - name: load-data-gpdb6
   246    plan:
   247    - in_parallel:
   248      - get: weekly-trigger
   249        trigger: true
   250      - get: rocky8-gpdb6-image
   251      - get: gpbackup
   252      - get: gpdb_binary
   253        resource: bin_gpdb_6x_rhel8
   254      - get: gpdb_src
   255        resource: gpdb6_src
   256      - get: gppkgs
   257      - get: ccp_src
   258      - get: terraform.d
   259        params:
   260          unpack: true
   261    - put: terraform
   262      params:
   263        <<: *ccp_default_params
   264        terraform_source: ccp_src/google-nvme-block-device/
   265        vars:
   266          PLATFORM: rocky8
   267          instance_type: n2-standard-8
   268          disk_size: 1000
   269          number_of_nodes: 4
   270          segments_per_host: 4
   271    - task: gen_cluster
   272      params:
   273        <<: *ccp_gen_cluster_default_params
   274        PLATFORM: rocky8
   275      file: ccp_src/ci/tasks/gen_cluster.yml
   276    - task: gpinitsystem
   277      file: ccp_src/ci/tasks/gpinitsystem.yml
   278    - task: setup-cluster-env
   279      image: rocky8-gpdb6-image
   280      file: gpbackup/ci/tasks/setup-cluster-env.yml
   281    - task: scale-perf-load
   282      image: rocky8-gpdb6-image
   283      file: gpbackup/ci/tasks/scale-perf-load.yml
   284      params:
   285        GOOGLE_CREDENTIALS: ((dp/dev/google-service-account-key))
   286      on_failure:
   287        *slack_alert
   288    - put: cluster-metadata
   289      params:
   290        file: cluster-metadata/cluster-metadata.tar.gz
   291  
   292  - name: scale-perf-tests-gpdb6
   293    plan:
   294    - in_parallel:
   295      - get: cluster-metadata
   296        passed: ['load-data-gpdb6']
   297        trigger: true
   298      - get: rocky8-gpdb6-image
   299      - get: gpbackup
   300      - get: gpdb_binary
   301        resource: bin_gpdb_6x_rhel8
   302      - get: gpdb_src
   303        resource: gpdb6_src
   304      - get: gppkgs
   305      - get: ccp_src
   306    - task: scale-perf-tests
   307      image: rocky8-gpdb6-image
   308      file: gpbackup/ci/tasks/scale-perf-tests.yml
   309      params:
   310        RESULTS_LOG_FILE: /tmp/gpbackup.log
   311        RESULTS_DATABASE_HOST: 10.122.32.4
   312        RESULTS_DATABASE_USER: postgres
   313        RESULTS_DATABASE_NAME: gpbackup_refdb
   314        RESULTS_DATABASE_PASSWORD: ((dp/dev/gcp_refdb_admin_password))
   315      on_failure:
   316        *slack_alert
   317    - task: scale-test-slack-notify
   318      image: rocky8-gpdb6-image
   319      file: gpbackup/ci/tasks/scale-tests-slack-notify.yml
   320      params:
   321        RESULTS_DATABASE_HOST: 10.122.32.4
   322        RESULTS_DATABASE_USER: postgres
   323        RESULTS_DATABASE_NAME: gpbackup_refdb
   324        SLACK_WEBHOOK_URL: ((dp/webhook_url))
   325