github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/.github/workflows/upgrade_dm_via_tiup.yaml (about)

     1  name: Upgrade DM via TiUP
     2  
     3  on:
     4    schedule:
     5      - cron: '0 17-23 * * *' # run at minute 0 every hour from 01:00 ~ 07:00 UTC+8
     6  
     7    workflow_dispatch:
     8      inputs:
     9        pr:
    10          description: 'Which PR do you want to trigger'
    11          required: true
    12          default: ''
    13        # TODO: use these inputs
    14        fromVer:
    15          dmVer: "v1.0.7"
    16        toVer:
    17          dmVer: "nightly"
    18  
    19  # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
    20  concurrency:
    21    group: ${{ github.ref }}-${{ github.workflow }}
    22    cancel-in-progress: true
    23  
    24  jobs:
    25    from_v1:
    26      name: From V1
    27      runs-on: ubuntu-20.04
    28      # skip it now, the ansible script fails with "Could not find the requested service prometheus-9090.service: host"
    29      # TODO enable it later.
    30      if: ${{ false }}
    31      env:
    32        working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/tiflow
    33  
    34      steps:
    35        - name: Set up Go env
    36          uses: actions/setup-go@v3
    37          with:
    38            go-version: '1.21'
    39  
    40        - name: Check out code
    41          uses: actions/checkout@v2
    42          with:
    43            path: go/src/github.com/pingcap/tiflow
    44  
    45        - name: Check out code by workflow dispatch
    46          if: ${{ github.event.inputs.pr != '' }}
    47          uses: actions/checkout@v2
    48          with:
    49            path: go/src/github.com/pingcap/tiflow
    50            ref: refs/pull/${{ github.event.inputs.pr }}/head
    51  
    52        - name: Setup containers
    53          working-directory: ${{ env.working-directory }}
    54          run: |
    55            cd ${{ env.working-directory }}/dm/tests/tiup/docker
    56            GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d
    57  
    58        - name: Run test cases
    59          working-directory: ${{ env.working-directory }}
    60          run: |
    61            cd ${{ env.working-directory }}/dm/tests/tiup/docker
    62            docker-compose exec -T control bash -c "cd /go/src/github.com/pingcap/tiflow/dm && ./tests/tiup/upgrade-from-v1.sh"
    63  
    64        # send Slack notify if failed.
    65        # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
    66        - name: Slack notification
    67          if: ${{ failure() }}
    68          env:
    69            SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }}
    70          uses: Ilshidur/action-slack@2.1.0
    71          with:
    72            args: "upgrade job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"
    73  
    74    from_v2:
    75      name: From V2
    76      runs-on: ubuntu-20.04
    77      env:
    78        working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/tiflow
    79      strategy:
    80        fail-fast: false
    81        matrix:
    82          previous_v2:
    83            ["v2.0.1", "v2.0.7", "v5.3.0"]
    84  
    85      steps:
    86        - name: Set up Go env
    87          uses: actions/setup-go@v3
    88          with:
    89            go-version: '1.21'
    90  
    91        - name: Check out code
    92          uses: actions/checkout@v2
    93          with:
    94            path: go/src/github.com/pingcap/tiflow
    95  
    96        - name: Check out code by workflow dispatch
    97          if: ${{ github.event.inputs.pr != '' }}
    98          uses: actions/checkout@v2
    99          with:
   100            path: go/src/github.com/pingcap/tiflow
   101            ref: refs/pull/${{ github.event.inputs.pr }}/head
   102  
   103        - name: Build
   104          if: ${{ github.ref != 'refs/heads/master' || github.event.inputs.pr != '' }}
   105          working-directory: ${{ env.working-directory }}
   106          run: make dm
   107  
   108        - name: Package files
   109          if: ${{ github.ref != 'refs/heads/master' || github.event.inputs.pr != '' }}
   110          run: |
   111            mkdir ${{ github.workspace }}/package
   112            cd ${{ github.workspace }}/package
   113  
   114            echo "package dm-master"
   115            mkdir dm-master
   116            cp ${{ env.working-directory }}/bin/dm-master dm-master
   117            cp -r ${{ env.working-directory }}/metrics/grafana/ dm-master/scripts
   118            cp -r ${{ env.working-directory }}/dm/metrics/alertmanager/ dm-master/conf
   119            tar -czvf dm-master-nightly-linux-amd64.tar.gz dm-master
   120  
   121            echo "package dm-worker"
   122            mkdir dm-worker
   123            cp ${{ env.working-directory }}/bin/dm-worker dm-worker
   124            cp -r ${{ env.working-directory }}/metrics/grafana/ dm-worker/scripts
   125            cp -r ${{ env.working-directory }}/dm/metrics/alertmanager/ dm-worker/conf
   126            tar -czvf dm-worker-nightly-linux-amd64.tar.gz dm-worker
   127  
   128            echo "package dmctl"
   129            mkdir dmctl
   130            cp ${{ env.working-directory }}/bin/dmctl dmctl
   131            cp -r ${{ env.working-directory }}/metrics/grafana/ dmctl/scripts
   132            cp -r ${{ env.working-directory }}/dm/metrics/alertmanager/ dmctl/conf
   133            tar -czvf dmctl-nightly-linux-amd64.tar.gz dmctl
   134  
   135        - name: Setup containers
   136          working-directory: ${{ env.working-directory }}
   137          run: |
   138            cd ${{ env.working-directory }}/dm/tests/tiup/docker
   139            GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d
   140  
   141        - name: Copy package files
   142          if: ${{ github.ref != 'refs/heads/master' || github.event.inputs.pr != '' }}
   143          run: |
   144            cd ${{ github.workspace }}/package
   145            docker cp dm-master-nightly-linux-amd64.tar.gz control:/tmp
   146            docker cp dm-worker-nightly-linux-amd64.tar.gz control:/tmp
   147            docker cp dmctl-nightly-linux-amd64.tar.gz control:/tmp
   148  
   149        # TODO: support more CUR_VER
   150        - name: Run test cases
   151          working-directory: ${{ env.working-directory }}
   152          run: |
   153            cd ${{ env.working-directory }}/dm/tests/tiup/docker
   154            docker-compose exec -e ref=${{ github.ref }} -e id=${{ github.event.inputs.pr }} -T control bash -c "cd /go/src/github.com/pingcap/tiflow/dm && ./tests/tiup/upgrade-from-v2.sh ${{ matrix.previous_v2 }} nightly"
   155  
   156        # if above step is passed, logs will be removed by tiup dm destroy
   157        - name: Copy logs to hack permission
   158          if: ${{ failure() }}
   159          run: |
   160            mkdir ./logs
   161            docker cp -L master1:/home/tidb/dm/deploy/dm-master-8261/log ./logs/master
   162            docker cp -L worker1:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker1
   163            docker cp -L worker2:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker2
   164            docker cp -L control:/tmp/tiup-dm-operation.log ./logs
   165            sudo chown -R runner ./logs
   166  
   167        # Update logs as artifact seems not stable, so we set `continue-on-error: true` here.
   168        - name: Upload logs
   169          continue-on-error: true
   170          uses: actions/upload-artifact@v2
   171          if: ${{ failure() }}
   172          with:
   173            name: upgrade-via-tiup-${{ matrix.previous_v2 }}
   174            path: |
   175              ./logs
   176  
   177        # send Slack notify if failed.
   178        # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
   179        - name: Slack notification
   180          if: ${{ failure() }}
   181          env:
   182            SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }}
   183          uses: Ilshidur/action-slack@2.1.0
   184          with:
   185            args: "upgrade-via-tiup job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"
   186  
   187    upgrade_tidb:
   188      name: Upgrade TiDB
   189      runs-on: ubuntu-20.04
   190      env:
   191        working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/tiflow
   192      steps:
   193        - name: Set up Go env
   194          uses: actions/setup-go@v3
   195          with:
   196            go-version: '1.21'
   197  
   198        - name: Check out code
   199          uses: actions/checkout@v2
   200          with:
   201            path: go/src/github.com/pingcap/tiflow
   202  
   203        # TODO: support more versions
   204        - name: Setup containers
   205          working-directory: ${{ env.working-directory }}
   206          run: |
   207            cd ${{ env.working-directory }}/dm/tests/tiup/docker
   208            sed -i "s/tidb:v4.0.7/tidb:v3.0.19/g" docker-compose.yml
   209            GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d
   210  
   211        - name: Run test cases before upgrade
   212          working-directory: ${{ env.working-directory }}
   213          run: |
   214            cd ${{ env.working-directory }}/dm/tests/tiup/docker
   215            docker-compose exec -T control bash -c "cd /go/src/github.com/pingcap/tiflow && ./dm/tests/tiup/upgrade-tidb.sh before_upgrade nightly"
   216  
   217        - name: Upgrade TiDB
   218          working-directory: ${{ env.working-directory }}
   219          run: |
   220            cd ${{ env.working-directory }}/dm/tests/tiup/docker
   221            sed -i "s/tidb:v3.0.19/tidb:v4.0.7/g" docker-compose.yml
   222            GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d
   223  
   224        - name: Run test cases after upgrade
   225          working-directory: ${{ env.working-directory }}
   226          run: |
   227            cd ${{ env.working-directory }}/dm/tests/tiup/docker
   228            docker-compose exec -T control bash -c "source /root/.profile && cd /go/src/github.com/pingcap/tiflow && ./dm/tests/tiup/upgrade-tidb.sh after_upgrade nightly"
   229  
   230        # send Slack notify if failed.
   231        # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
   232        - name: Slack notification
   233          if: ${{ failure() }}
   234          env:
   235            SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }}
   236          uses: Ilshidur/action-slack@2.1.0
   237          with:
   238            args: "upgrade job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"