github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/scripts/jenkins_ci/integration_test_common.groovy (about)

     1  CONCURRENT_NUMBER = 8
     2  
     3  def prepare_binaries() {
     4      stage('Prepare Binaries') {
     5          def prepares = [:]
     6  
     7          prepares["build binaries"] = {
     8              node ("${GO_TEST_SLAVE}") {
     9                  container("golang") {
    10                      println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
    11                      def ws = pwd()
    12                      deleteDir()
    13                      unstash 'ticdc'
    14  
    15                      dir("go/src/github.com/pingcap/ticdc") {
    16                          sh """
    17                              GO111MODULE=off GOPATH=\$GOPATH:${ws}/go PATH=\$GOPATH/bin:${ws}/go/bin:\$PATH make cdc
    18                              GO111MODULE=off GOPATH=\$GOPATH:${ws}/go PATH=\$GOPATH/bin:${ws}/go/bin:\$PATH make integration_test_build
    19                              GO111MODULE=off GOPATH=\$GOPATH:${ws}/go PATH=\$GOPATH/bin:${ws}/go/bin:\$PATH make kafka_consumer
    20                              GO111MODULE=off GOPATH=\$GOPATH:${ws}/go PATH=\$GOPATH/bin:${ws}/go/bin:\$PATH make check_failpoint_ctl
    21                              tar czvf ticdc_bin.tar.gz bin/*
    22                              curl -F test/cdc/ci/ticdc_bin_${env.BUILD_NUMBER}.tar.gz=@ticdc_bin.tar.gz http://fileserver.pingcap.net/upload
    23                          """
    24                      }
    25                      dir("go/src/github.com/pingcap/ticdc/tests") {
    26                          def cases_name = sh (
    27                              script: 'find . -maxdepth 2 -mindepth 2 -name \'run.sh\' | awk -F/ \'{print $2}\'',
    28                              returnStdout: true
    29                          ).trim().split().join(" ")
    30                          sh "echo ${cases_name} > CASES"
    31                      }
    32                      stash includes: "go/src/github.com/pingcap/ticdc/tests/CASES", name: "cases_name", useDefaultExcludes: false
    33                  }
    34              }
    35          }
    36          parallel prepares
    37      }
    38  }
    39  
    40  def tests(sink_type, node_label) {
    41      stage("Tests") {
    42          def test_cases = [:]
    43  
    44          test_cases["unit test"] = {
    45              node (node_label) {
    46                  container("golang") {
    47                      def ws = pwd()
    48                      deleteDir()
    49                      println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
    50                      println "work space path:\n${ws}"
    51                      unstash 'ticdc'
    52  
    53                      dir("go/src/github.com/pingcap/ticdc") {
    54                          sh """
    55                              rm -rf /tmp/tidb_cdc_test
    56                              mkdir -p /tmp/tidb_cdc_test
    57                              GO111MODULE=off GOPATH=\$GOPATH:${ws}/go PATH=\$GOPATH/bin:${ws}/go/bin:\$PATH make test
    58                              rm -rf cov_dir
    59                              mkdir -p cov_dir
    60                              ls /tmp/tidb_cdc_test
    61                              cp /tmp/tidb_cdc_test/cov*out cov_dir
    62                          """
    63                          sh """
    64                          tail /tmp/tidb_cdc_test/cov*
    65                          """
    66                      }
    67                      stash includes: "go/src/github.com/pingcap/ticdc/cov_dir/**", name: "unit_test", useDefaultExcludes: false
    68                  }
    69              }
    70          }
    71  
    72          def run_integration_test = { step_name, case_names ->
    73              node (node_label) {
    74                  container("golang") {
    75                      def ws = pwd()
    76                      deleteDir()
    77                      println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
    78                      println "work space path:\n${ws}"
    79                      println "this step will run tests: ${case_names}"
    80                      unstash 'ticdc'
    81                      dir("go/src/github.com/pingcap/ticdc") {
    82                          download_binaries()
    83                          try {
    84                              sh """
    85                                  sudo pip install s3cmd
    86                                  rm -rf /tmp/tidb_cdc_test
    87                                  mkdir -p /tmp/tidb_cdc_test
    88                                  echo "${env.KAFKA_VERSION}" > /tmp/tidb_cdc_test/KAFKA_VERSION
    89                                  GO111MODULE=off GOPATH=\$GOPATH:${ws}/go PATH=\$GOPATH/bin:${ws}/go/bin:\$PATH make integration_test_${sink_type} CASE="${case_names}"
    90                                  rm -rf cov_dir
    91                                  mkdir -p cov_dir
    92                                  ls /tmp/tidb_cdc_test
    93                                  cp /tmp/tidb_cdc_test/cov*out cov_dir || touch cov_dir/dummy_file_${step_name}
    94                              """
    95                              // cyclic tests do not run on kafka sink, so there is no cov* file.
    96                              sh """
    97                              tail /tmp/tidb_cdc_test/cov* || true
    98                              """
    99                          } catch (Exception e) {
   100                              sh """
   101                                  echo "archive all log"
   102                                  for log in `ls /tmp/tidb_cdc_test/*/*.log`; do
   103                                      dirname=`dirname \$log`
   104                                      basename=`basename \$log`
   105                                      mkdir -p "log\$dirname"
   106                                      tar zcvf "log\${log}.tgz" -C "\$dirname" "\$basename"
   107                                  done
   108                              """
   109                              archiveArtifacts artifacts: "log/tmp/tidb_cdc_test/**/*.tgz", caseSensitive: false
   110                              throw e;
   111                          }
   112                      }
   113                      stash includes: "go/src/github.com/pingcap/ticdc/cov_dir/**", name: "integration_test_${step_name}", useDefaultExcludes: false
   114                  }
   115              }
   116          }
   117  
   118  
   119          unstash 'cases_name'
   120          def cases_name = sh (
   121              script: 'cat go/src/github.com/pingcap/ticdc/tests/CASES',
   122              returnStdout: true
   123          ).trim().split()
   124  
   125          def step_cases = []
   126          def step_length = (int)(cases_name.size() / CONCURRENT_NUMBER + 0.5)
   127          for(int i in 1..CONCURRENT_NUMBER) {
   128              def end = i*step_length-1
   129              if (i == CONCURRENT_NUMBER){
   130                  end = cases_name.size()-1
   131              }
   132              step_cases.add(cases_name[(i-1)*step_length..end])
   133          }
   134          step_cases.eachWithIndex{ case_names, index ->
   135              def step_name = "step_${index}"
   136              test_cases["integration test ${step_name}"] = {
   137                  run_integration_test(step_name, case_names.join(" "))
   138              }
   139          }
   140  
   141          parallel test_cases
   142      }
   143  }
   144  
   145  def download_binaries(){
   146      def TIDB_BRANCH = params.getOrDefault("release_test__tidb_commit", "master")
   147      def TIKV_BRANCH = params.getOrDefault("release_test__tikv_commit", "master")
   148      def PD_BRANCH = params.getOrDefault("release_test__pd_commit", "master")
   149      // TODO master tiflash release version is 4.1.0-rc-43-gxxx, which is not compatible
   150      // with TiKV 5.0.0-rc.x. Change default branch to master after TiFlash fixes it.
   151      def TIFLASH_BRANCH = params.getOrDefault("release_test__release_branch", "release-5.0-rc")
   152      def TIFLASH_COMMIT = params.getOrDefault("release_test__tiflash_commit", null)
   153  
   154      // parse tidb branch
   155      def m1 = ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/
   156      if (m1) {
   157          TIDB_BRANCH = "${m1[0][1]}"
   158      }
   159      m1 = null
   160      println "TIDB_BRANCH=${TIDB_BRANCH}"
   161  
   162      // parse tikv branch
   163      def m2 = ghprbCommentBody =~ /tikv\s*=\s*([^\s\\]+)(\s|\\|$)/
   164      if (m2) {
   165          TIKV_BRANCH = "${m2[0][1]}"
   166      }
   167      m2 = null
   168      println "TIKV_BRANCH=${TIKV_BRANCH}"
   169  
   170      // parse pd branch
   171      def m3 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/
   172      if (m3) {
   173          PD_BRANCH = "${m3[0][1]}"
   174      }
   175      m3 = null
   176      println "PD_BRANCH=${PD_BRANCH}"
   177  
   178      // parse tiflash branch
   179      def m4 = ghprbCommentBody =~ /tiflash\s*=\s*([^\s\\]+)(\s|\\|$)/
   180      if (m4) {
   181          TIFLASH_BRANCH = "${m4[0][1]}"
   182      }
   183      m4 = null
   184      println "TIFLASH_BRANCH=${TIFLASH_BRANCH}"
   185  
   186      println "debug command:\nkubectl -n jenkins-ci exec -ti ${NODE_NAME} bash"
   187      def tidb_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tidb/${TIDB_BRANCH}/sha1").trim()
   188      def tikv_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim()
   189      def pd_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim()
   190      def tiflash_sha1 = TIFLASH_COMMIT
   191      if (TIFLASH_COMMIT == null) {
   192          tiflash_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tiflash/${TIFLASH_BRANCH}/sha1").trim()
   193      }
   194      sh """
   195          mkdir -p third_bin
   196          mkdir -p tmp
   197          mkdir -p bin
   198  
   199          tidb_url="${FILE_SERVER_URL}/download/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz"
   200          tikv_url="${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz"
   201          pd_url="${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz"
   202          tiflash_url="${FILE_SERVER_URL}/download/builds/pingcap/tiflash/${TIFLASH_BRANCH}/${tiflash_sha1}/centos7/tiflash.tar.gz"
   203          minio_url="${FILE_SERVER_URL}/download/minio.tar.gz"
   204  
   205          curl \${tidb_url} | tar xz -C ./tmp bin/tidb-server
   206          curl \${pd_url} | tar xz -C ./tmp bin/*
   207          curl \${tikv_url} | tar xz -C ./tmp bin/tikv-server
   208          curl \${minio_url} | tar xz -C ./tmp/bin minio
   209          mv tmp/bin/* third_bin
   210          curl \${tiflash_url} | tar xz -C third_bin
   211          mv third_bin/tiflash third_bin/_tiflash
   212          mv third_bin/_tiflash/* third_bin
   213          curl ${FILE_SERVER_URL}/download/builds/pingcap/go-ycsb/test-br/go-ycsb -o third_bin/go-ycsb
   214          curl -L http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz | tar xz -C ./tmp
   215          mv tmp/etcd-v3.4.7-linux-amd64/etcdctl third_bin
   216          curl http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector.tar.gz | tar xz -C ./third_bin
   217          curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o jq
   218          mv jq third_bin
   219          chmod a+x third_bin/*
   220          rm -rf tmp
   221          curl -L http://fileserver.pingcap.net/download/test/cdc/ci/ticdc_bin_${env.BUILD_NUMBER}.tar.gz | tar xvz -C .
   222          mv ./third_bin/* ./bin
   223          rm -rf third_bin
   224      """
   225  }
   226  
   227  def coverage() {
   228      stage('Coverage') {
   229          node("${GO_TEST_SLAVE}") {
   230              def ws = pwd()
   231              deleteDir()
   232              unstash 'ticdc'
   233              unstash 'unit_test'
   234              for(int i in 1..CONCURRENT_NUMBER) {
   235                  unstash "integration_test_step_${i-1}"
   236              }
   237  
   238              dir("go/src/github.com/pingcap/ticdc") {
   239                  container("golang") {
   240                      archiveArtifacts artifacts: 'cov_dir/*', fingerprint: true
   241  
   242                      timeout(30) {
   243                          sh """
   244                          rm -rf /tmp/tidb_cdc_test
   245                          mkdir -p /tmp/tidb_cdc_test
   246                          cp cov_dir/* /tmp/tidb_cdc_test
   247                          set +x
   248                          BUILD_NUMBER=${env.BUILD_NUMBER} CODECOV_TOKEN="${CODECOV_TOKEN}" COVERALLS_TOKEN="${COVERALLS_TOKEN}" GOPATH=${ws}/go:\$GOPATH PATH=${ws}/go/bin:/go/bin:\$PATH JenkinsCI=1 make coverage
   249                          set -x
   250                          """
   251                      }
   252                  }
   253              }
   254          }
   255      }
   256  }
   257  
   258  return this