github.com/percona/percona-xtradb-cluster-operator@v1.14.0/Jenkinsfile (about)

     1  region='us-central1-a'
     2  testUrlPrefix="https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator"
     3  tests=[]
     4  
     5  void createCluster(String CLUSTER_SUFFIX) {
     6      withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) {
     7          sh """
     8              NODES_NUM=3
     9              export KUBECONFIG=/tmp/$CLUSTER_NAME-${CLUSTER_SUFFIX}
    10              ret_num=0
    11              while [ \${ret_num} -lt 15 ]; do
    12                  ret_val=0
    13                  gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
    14                  gcloud config set project $GCP_PROJECT
    15                  gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
    16  
    17                  gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.25 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \
    18                  kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
    19                  if [ \${ret_val} -eq 0 ]; then break; fi
    20                  ret_num=\$((ret_num + 1))
    21              done
    22              if [ \${ret_num} -eq 15 ]; then
    23                  gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
    24                  exit 1
    25              fi
    26          """
    27     }
    28  }
    29  
    30  void shutdownCluster(String CLUSTER_SUFFIX) {
    31      withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) {
    32          sh """
    33              export KUBECONFIG=/tmp/$CLUSTER_NAME-${CLUSTER_SUFFIX}
    34              gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
    35              gcloud config set project $GCP_PROJECT
    36              for namespace in \$(kubectl get namespaces --no-headers | awk '{print \$1}' | grep -vE "^kube-|^openshift" | sed '/-operator/ s/^/1-/' | sort | sed 's/^1-//'); do
    37                  kubectl delete deployments --all -n \$namespace --force --grace-period=0 || true
    38                  kubectl delete sts --all -n \$namespace --force --grace-period=0 || true
    39                  kubectl delete replicasets --all -n \$namespace --force --grace-period=0 || true
    40                  kubectl delete poddisruptionbudget --all -n \$namespace --force --grace-period=0 || true
    41                  kubectl delete services --all -n \$namespace --force --grace-period=0 || true
    42                  kubectl delete pods --all -n \$namespace --force --grace-period=0 || true
    43              done
    44              kubectl get svc --all-namespaces || true
    45              gcloud container clusters delete --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX}
    46          """
    47     }
    48  }
    49  
    50  void deleteOldClusters(String FILTER) {
    51      withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) {
    52          sh """
    53              if gcloud --version > /dev/null 2>&1; then
    54                  gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
    55                  gcloud config set project $GCP_PROJECT
    56                  for GKE_CLUSTER in \$(gcloud container clusters list --format='csv[no-heading](name)' --filter="$FILTER"); do
    57                      GKE_CLUSTER_STATUS=\$(gcloud container clusters list --format='csv[no-heading](status)' --filter="\$GKE_CLUSTER")
    58                      retry=0
    59                      while [ "\$GKE_CLUSTER_STATUS" == "PROVISIONING" ]; do
    60                          echo "Cluster \$GKE_CLUSTER is being provisioned, waiting before delete."
    61                          sleep 10
    62                          GKE_CLUSTER_STATUS=\$(gcloud container clusters list --format='csv[no-heading](status)' --filter="\$GKE_CLUSTER")
    63                          let retry+=1
    64                          if [ \$retry -ge 60 ]; then
    65                              echo "Cluster \$GKE_CLUSTER to delete is being provisioned for too long. Skipping..."
    66                              break
    67                          fi
    68                      done
    69                      gcloud container clusters delete --async --zone $region --quiet \$GKE_CLUSTER || true
    70                  done
    71              fi
    72          """
    73     }
    74  }
    75  
    76  void pushLogFile(String FILE_NAME) {
    77      def LOG_FILE_PATH="e2e-tests/logs/${FILE_NAME}.log"
    78      def LOG_FILE_NAME="${FILE_NAME}.log"
    79      echo "Push logfile $LOG_FILE_NAME file to S3!"
    80      withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
    81          sh """
    82              S3_PATH=s3://percona-jenkins-artifactory-public/\$JOB_NAME/\$(git rev-parse --short HEAD)
    83              aws s3 ls \$S3_PATH/${LOG_FILE_NAME} || :
    84              aws s3 cp --content-type text/plain --quiet ${LOG_FILE_PATH} \$S3_PATH/${LOG_FILE_NAME} || :
    85          """
    86      }
    87  }
    88  
    89  void pushArtifactFile(String FILE_NAME) {
    90      echo "Push $FILE_NAME file to S3!"
    91  
    92      withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
    93          sh """
    94              touch ${FILE_NAME}
    95              S3_PATH=s3://percona-jenkins-artifactory/\$JOB_NAME/\$(git rev-parse --short HEAD)
    96              aws s3 ls \$S3_PATH/${FILE_NAME} || :
    97              aws s3 cp --quiet ${FILE_NAME} \$S3_PATH/${FILE_NAME} || :
    98          """
    99      }
   100  }
   101  
   102  void initTests() {
   103      echo "Populating tests into the tests array!"
   104  
   105      def records = readCSV file: 'e2e-tests/run-pr.csv'
   106  
   107      for (int i=0; i<records.size(); i++) {
   108          tests.add(["name": records[i][0], "mysql_ver": records[i][1], "cluster": "NA", "result": "skipped", "time": "0"])
   109      }
   110  
   111      markPassedTests()
   112  }
   113  
   114  void markPassedTests() {
   115      echo "Marking passed tests in the tests map!"
   116  
   117      withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
   118          sh """
   119              aws s3 ls "s3://percona-jenkins-artifactory/${JOB_NAME}/${env.GIT_SHORT_COMMIT}/" || :
   120          """
   121  
   122          for (int i=0; i<tests.size(); i++) {
   123              def testNameWithMysqlVersion = tests[i]["name"] +"-"+ tests[i]["mysql_ver"].replace(".", "-")
   124              def file="${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testNameWithMysqlVersion"
   125              def retFileExists = sh(script: "aws s3api head-object --bucket percona-jenkins-artifactory --key ${JOB_NAME}/${env.GIT_SHORT_COMMIT}/${file} >/dev/null 2>&1", returnStatus: true)
   126  
   127              if (retFileExists == 0) {
   128                  tests[i]["result"] = "passed"
   129              }
   130          }
   131      }
   132  }
   133  
   134  void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) {
   135      sh """
   136          export KUBECONFIG=/tmp/$CLUSTER_NAME-$CLUSTER_SUFFIX
   137          echo "========== KUBERNETES STATUS $LOCATION TEST =========="
   138          gcloud container clusters list|grep -E "NAME|$CLUSTER_NAME-$CLUSTER_SUFFIX "
   139          echo
   140          kubectl get nodes
   141          echo
   142          kubectl top nodes
   143          echo
   144          kubectl get pods --all-namespaces
   145          echo
   146          kubectl top pod --all-namespaces
   147          echo
   148          kubectl get events --field-selector type!=Normal --all-namespaces
   149          echo "======================================================"
   150      """
   151  }
   152  
   153  TestsReport = '| Test name  | Status |\r\n| ------------- | ------------- |'
   154  TestsReportXML = '<testsuite name=\\"PXC\\">\n'
   155  
   156  void makeReport() {
   157      def wholeTestAmount=tests.size()
   158      def startedTestAmount = 0
   159  
   160      for (int i=0; i<tests.size(); i++) {
   161          def testResult = tests[i]["result"]
   162          def testTime = tests[i]["time"]
   163          def testNameWithMysqlVersion = tests[i]["name"] +"-"+ tests[i]["mysql_ver"].replace(".", "-")
   164          def testUrl = "${testUrlPrefix}/${env.GIT_BRANCH}/${env.GIT_SHORT_COMMIT}/${testNameWithMysqlVersion}.log"
   165  
   166          if (tests[i]["result"] != "skipped") {
   167              startedTestAmount++
   168          }
   169          TestsReport = TestsReport + "\r\n| "+ testNameWithMysqlVersion +" | ["+ tests[i]["result"] +"]("+ testUrl +") |"
   170          TestsReportXML = TestsReportXML + '<testcase name=\\"' + testNameWithMysqlVersion + '\\" time=\\"' + testTime + '\\"><'+ testResult +'/></testcase>\n'
   171      }
   172      TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount|"
   173      TestsReportXML = TestsReportXML + '</testsuite>\n'
   174  }
   175  
   176  void clusterRunner(String cluster) {
   177      def clusterCreated=0
   178  
   179      for (int i=0; i<tests.size(); i++) {
   180          if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
   181              tests[i]["result"] = "failure"
   182              tests[i]["cluster"] = cluster
   183              if (clusterCreated == 0) {
   184                  createCluster(cluster)
   185                  clusterCreated++
   186              }
   187              runTest(i)
   188          }
   189      }
   190  
   191      if (clusterCreated >= 1) {
   192          shutdownCluster(cluster)
   193      }
   194  }
   195  
   196  void runTest(Integer TEST_ID) {
   197      def retryCount = 0
   198      def testName = tests[TEST_ID]["name"]
   199      def mysqlVer = tests[TEST_ID]["mysql_ver"]
   200      def clusterSuffix = tests[TEST_ID]["cluster"]
   201      def testNameWithMysqlVersion = "$testName-$mysqlVer".replace(".", "-")
   202  
   203      waitUntil {
   204          def timeStart = new Date().getTime()
   205          try {
   206              echo "The $testName-$mysqlVer test was started on cluster $CLUSTER_NAME-$clusterSuffix !"
   207              tests[TEST_ID]["result"] = "failure"
   208  
   209              timeout(time: 90, unit: 'MINUTES') {
   210                  sh """
   211                      if [ $retryCount -eq 0 ]; then
   212                          export DEBUG_TESTS=0
   213                      else
   214                          export DEBUG_TESTS=1
   215                      fi
   216                      export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix
   217                      export MYSQL_VERSION=$mysqlVer
   218                      time bash e2e-tests/$testName/run
   219                  """
   220              }
   221              pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testNameWithMysqlVersion")
   222              tests[TEST_ID]["result"] = "passed"
   223              return true
   224          }
   225          catch (exc) {
   226              printKubernetesStatus("AFTER","$clusterSuffix")
   227              if (retryCount >= 1 || currentBuild.nextBuild != null) {
   228                  currentBuild.result = 'FAILURE'
   229                  return true
   230              }
   231              retryCount++
   232              return false
   233          }
   234          finally {
   235              def timeStop = new Date().getTime()
   236              def durationSec = (timeStop - timeStart) / 1000
   237              tests[TEST_ID]["time"] = durationSec
   238              pushLogFile("$testNameWithMysqlVersion")
   239              echo "The $testName-$mysqlVer test was finished!"
   240          }
   241      }
   242  }
   243  
   244  def skipBranchBuilds = true
   245  if (env.CHANGE_URL) {
   246      skipBranchBuilds = false
   247  }
   248  
   249  pipeline {
   250      environment {
   251          CLOUDSDK_CORE_DISABLE_PROMPTS = 1
   252          CLEAN_NAMESPACE = 1
   253          OPERATOR_NS = 'pxc-operator'
   254          GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', , returnStdout: true).trim()
   255          VERSION = "${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}"
   256          CLUSTER_NAME = sh(script: "echo jen-pxc-${env.CHANGE_ID}-${GIT_SHORT_COMMIT}-${env.BUILD_NUMBER} | tr '[:upper:]' '[:lower:]'", , returnStdout: true).trim()
   257          AUTHOR_NAME = sh(script: "echo ${CHANGE_AUTHOR_EMAIL} | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
   258          ENABLE_LOGGING = "true"
   259      }
   260      agent {
   261          label 'docker'
   262      }
   263      options {
   264          disableConcurrentBuilds(abortPrevious: true)
   265      }
   266      stages {
   267          stage('Prepare') {
   268              when {
   269                  expression {
   270                      !skipBranchBuilds
   271                  }
   272              }
   273              steps {
   274                  initTests()
   275                  script {
   276                      if (AUTHOR_NAME == 'null') {
   277                          AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
   278                      }
   279                      for (comment in pullRequest.comments) {
   280                          println("Author: ${comment.user}, Comment: ${comment.body}")
   281                          if (comment.user.equals('JNKPercona')) {
   282                              println("delete comment")
   283                              comment.delete()
   284                          }
   285                      }
   286                  }
   287                  sh """
   288                      sudo curl -s -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && sudo chmod +x /usr/local/bin/kubectl
   289                      kubectl version --client --output=yaml
   290  
   291                      curl -fsSL https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz | sudo tar -C /usr/local/bin --strip-components 1 -xzf - linux-amd64/helm
   292  
   293                      sudo sh -c "curl -s -L https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 > /usr/local/bin/yq"
   294                      sudo chmod +x /usr/local/bin/yq
   295  
   296                      sudo sh -c "curl -s -L https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 > /usr/local/bin/jq"
   297                      sudo chmod +x /usr/local/bin/jq
   298  
   299                      sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOF
   300  [google-cloud-cli]
   301  name=Google Cloud CLI
   302  baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
   303  enabled=1
   304  gpgcheck=1
   305  repo_gpgcheck=0
   306  gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
   307  EOF
   308                      sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
   309  
   310                      curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
   311  
   312                      sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true
   313                      sudo percona-release enable-only tools
   314                      sudo yum install -y percona-xtrabackup-80 | true
   315                  """
   316  
   317                  withCredentials([file(credentialsId: 'cloud-secret-file', variable: 'CLOUD_SECRET_FILE')]) {
   318                      sh '''
   319                          cp $CLOUD_SECRET_FILE e2e-tests/conf/cloud-secret.yml
   320                      '''
   321                  }
   322                  deleteOldClusters("jen-pxc-$CHANGE_ID")
   323              }
   324          }
   325          stage('Build docker image') {
   326              when {
   327                  expression {
   328                      !skipBranchBuilds
   329                  }
   330              }
   331              steps {
   332                  withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) {
   333                      sh '''
   334                          DOCKER_TAG=perconalab/percona-xtradb-cluster-operator:$VERSION
   335                          docker_tag_file='./results/docker/TAG'
   336                          mkdir -p $(dirname ${docker_tag_file})
   337                          echo ${DOCKER_TAG} > "${docker_tag_file}"
   338                              sg docker -c "
   339                                  docker login -u '${USER}' -p '${PASS}'
   340                                  export RELEASE=0
   341                                  export IMAGE=\$DOCKER_TAG
   342                                  docker buildx create --use
   343                                  ./e2e-tests/build
   344                                  docker logout
   345                              "
   346                          sudo rm -rf ./build
   347                      '''
   348                  }
   349                  stash includes: 'results/docker/TAG', name: 'IMAGE'
   350                  archiveArtifacts 'results/docker/TAG'
   351              }
   352          }
   353          stage('GoLicenseDetector test') {
   354              when {
   355                  expression {
   356                      !skipBranchBuilds
   357                  }
   358              }
   359              steps {
   360                  sh """
   361                      mkdir -p $WORKSPACE/src/github.com/percona
   362                      ln -s $WORKSPACE $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator
   363                      sg docker -c "
   364                          docker run \
   365                              --rm \
   366                              -v $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator:/go/src/github.com/percona/percona-xtradb-cluster-operator \
   367                              -w /go/src/github.com/percona/percona-xtradb-cluster-operator \
   368                              golang:1.21 sh -c '
   369                                  go install -mod=readonly github.com/google/go-licenses@latest;
   370                                  /go/bin/go-licenses csv github.com/percona/percona-xtradb-cluster-operator/cmd/manager \
   371                                      | cut -d , -f 3 \
   372                                      | sort -u \
   373                                      > go-licenses-new || :
   374                              '
   375                      "
   376                      diff -u e2e-tests/license/compare/go-licenses go-licenses-new
   377                  """
   378              }
   379          }
   380          stage('GoLicense test') {
   381              when {
   382                  expression {
   383                      !skipBranchBuilds
   384                  }
   385              }
   386              steps {
   387                  sh '''
   388                      mkdir -p $WORKSPACE/src/github.com/percona
   389                      ln -s $WORKSPACE $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator
   390                      sg docker -c "
   391                          docker run \
   392                              --rm \
   393                              -v $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator:/go/src/github.com/percona/percona-xtradb-cluster-operator \
   394                              -w /go/src/github.com/percona/percona-xtradb-cluster-operator \
   395                              -e GO111MODULE=on \
   396                              -e GOFLAGS='-buildvcs=false' \
   397                              golang:1.21 sh -c 'go build -v -o percona-xtradb-cluster-operator github.com/percona/percona-xtradb-cluster-operator/cmd/manager'
   398                      "
   399                  '''
   400  
   401                  withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'GITHUB_TOKEN')]) {
   402                      sh """
   403                          golicense -plain ./percona-xtradb-cluster-operator \
   404                              | grep -v 'license not found' \
   405                              | sed -r 's/^[^ ]+[ ]+//' \
   406                              | sort \
   407                              | uniq \
   408                              > golicense-new || true
   409                          diff -u e2e-tests/license/compare/golicense golicense-new
   410                      """
   411                  }
   412              }
   413          }
   414          stage('Run tests for operator') {
   415              when {
   416                  expression {
   417                      !skipBranchBuilds
   418                  }
   419              }
   420              options {
   421                  timeout(time: 4, unit: 'HOURS')
   422              }
   423              parallel {
   424                  stage('cluster1') {
   425                      steps {
   426                          clusterRunner('cluster1')
   427                      }
   428                  }
   429                  stage('cluster2') {
   430                      steps {
   431                          clusterRunner('cluster2')
   432                      }
   433                  }
   434                  stage('cluster3') {
   435                      steps {
   436                          clusterRunner('cluster3')
   437                      }
   438                  }
   439                  stage('cluster4') {
   440                      steps {
   441                          clusterRunner('cluster4')
   442                      }
   443                  }
   444                  stage('cluster5') {
   445                      steps {
   446                          clusterRunner('cluster5')
   447                      }
   448                  }
   449                  stage('cluster6') {
   450                      steps {
   451                          clusterRunner('cluster6')
   452                      }
   453                  }
   454                  stage('cluster7') {
   455                      steps {
   456                          clusterRunner('cluster7')
   457                      }
   458                  }
   459                  stage('cluster8') {
   460                      steps {
   461                          clusterRunner('cluster8')
   462                      }
   463                  }
   464                  stage('cluster9') {
   465                      steps {
   466                          clusterRunner('cluster9')
   467                      }
   468                  }
   469              }
   470          }
   471      }
   472      post {
   473          always {
   474              script {
   475                  echo "CLUSTER ASSIGNMENTS\n" + tests.toString().replace("], ","]\n").replace("]]","]").replaceFirst("\\[","")
   476  
   477                  if (currentBuild.result != null && currentBuild.result != 'SUCCESS' && currentBuild.nextBuild == null) {
   478                      try {
   479                          slackSend channel: "@${AUTHOR_NAME}", color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
   480                      }
   481                      catch (exc) {
   482                          slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
   483                      }
   484                  }
   485  
   486                  if (env.CHANGE_URL && currentBuild.nextBuild == null) {
   487                      for (comment in pullRequest.comments) {
   488                          println("Author: ${comment.user}, Comment: ${comment.body}")
   489                          if (comment.user.equals('JNKPercona')) {
   490                              println("delete comment")
   491                              comment.delete()
   492                          }
   493                      }
   494                      makeReport()
   495                      sh """
   496                          echo "${TestsReportXML}" > TestsReport.xml
   497                      """
   498                      step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
   499                      archiveArtifacts '*.xml'
   500  
   501                      unstash 'IMAGE'
   502                      def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
   503                      TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
   504                      pullRequest.comment(TestsReport)
   505                  }
   506              }
   507              deleteOldClusters("$CLUSTER_NAME")
   508              sh """
   509                  sudo docker system prune --volumes -af
   510                  sudo rm -rf *
   511              """
   512              deleteDir()
   513          }
   514      }
   515  }