github.com/percona/percona-xtradb-cluster-operator@v1.14.0/Jenkinsfile (about) 1 region='us-central1-a' 2 testUrlPrefix="https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator" 3 tests=[] 4 5 void createCluster(String CLUSTER_SUFFIX) { 6 withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) { 7 sh """ 8 NODES_NUM=3 9 export KUBECONFIG=/tmp/$CLUSTER_NAME-${CLUSTER_SUFFIX} 10 ret_num=0 11 while [ \${ret_num} -lt 15 ]; do 12 ret_val=0 13 gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE 14 gcloud config set project $GCP_PROJECT 15 gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true 16 17 gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.25 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \ 18 kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$? 19 if [ \${ret_val} -eq 0 ]; then break; fi 20 ret_num=\$((ret_num + 1)) 21 done 22 if [ \${ret_num} -eq 15 ]; then 23 gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true 24 exit 1 25 fi 26 """ 27 } 28 } 29 30 void shutdownCluster(String CLUSTER_SUFFIX) { 31 withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) { 32 sh """ 33 export KUBECONFIG=/tmp/$CLUSTER_NAME-${CLUSTER_SUFFIX} 34 gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE 35 gcloud config set project $GCP_PROJECT 36 for namespace in \$(kubectl get namespaces --no-headers | awk '{print \$1}' | grep -vE "^kube-|^openshift" | sed '/-operator/ s/^/1-/' | sort | sed 's/^1-//'); do 37 kubectl delete deployments --all -n \$namespace --force --grace-period=0 || true 38 kubectl delete sts --all -n \$namespace --force --grace-period=0 || true 39 kubectl delete replicasets --all -n \$namespace --force --grace-period=0 || true 40 kubectl delete poddisruptionbudget --all -n \$namespace --force --grace-period=0 || true 41 kubectl delete services --all -n \$namespace --force --grace-period=0 || true 42 kubectl delete pods --all -n \$namespace --force --grace-period=0 || true 43 done 44 kubectl get svc --all-namespaces || true 45 gcloud container clusters delete --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} 46 """ 47 } 48 } 49 50 void deleteOldClusters(String FILTER) { 51 withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) { 52 sh """ 53 if gcloud --version > /dev/null 2>&1; then 54 gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE 55 gcloud config set project $GCP_PROJECT 56 for GKE_CLUSTER in \$(gcloud container clusters list --format='csv[no-heading](name)' --filter="$FILTER"); do 57 GKE_CLUSTER_STATUS=\$(gcloud container clusters list --format='csv[no-heading](status)' --filter="\$GKE_CLUSTER") 58 retry=0 59 while [ "\$GKE_CLUSTER_STATUS" == "PROVISIONING" ]; do 60 echo "Cluster \$GKE_CLUSTER is being provisioned, waiting before delete." 61 sleep 10 62 GKE_CLUSTER_STATUS=\$(gcloud container clusters list --format='csv[no-heading](status)' --filter="\$GKE_CLUSTER") 63 let retry+=1 64 if [ \$retry -ge 60 ]; then 65 echo "Cluster \$GKE_CLUSTER to delete is being provisioned for too long. Skipping..." 66 break 67 fi 68 done 69 gcloud container clusters delete --async --zone $region --quiet \$GKE_CLUSTER || true 70 done 71 fi 72 """ 73 } 74 } 75 76 void pushLogFile(String FILE_NAME) { 77 def LOG_FILE_PATH="e2e-tests/logs/${FILE_NAME}.log" 78 def LOG_FILE_NAME="${FILE_NAME}.log" 79 echo "Push logfile $LOG_FILE_NAME file to S3!" 80 withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { 81 sh """ 82 S3_PATH=s3://percona-jenkins-artifactory-public/\$JOB_NAME/\$(git rev-parse --short HEAD) 83 aws s3 ls \$S3_PATH/${LOG_FILE_NAME} || : 84 aws s3 cp --content-type text/plain --quiet ${LOG_FILE_PATH} \$S3_PATH/${LOG_FILE_NAME} || : 85 """ 86 } 87 } 88 89 void pushArtifactFile(String FILE_NAME) { 90 echo "Push $FILE_NAME file to S3!" 91 92 withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { 93 sh """ 94 touch ${FILE_NAME} 95 S3_PATH=s3://percona-jenkins-artifactory/\$JOB_NAME/\$(git rev-parse --short HEAD) 96 aws s3 ls \$S3_PATH/${FILE_NAME} || : 97 aws s3 cp --quiet ${FILE_NAME} \$S3_PATH/${FILE_NAME} || : 98 """ 99 } 100 } 101 102 void initTests() { 103 echo "Populating tests into the tests array!" 104 105 def records = readCSV file: 'e2e-tests/run-pr.csv' 106 107 for (int i=0; i<records.size(); i++) { 108 tests.add(["name": records[i][0], "mysql_ver": records[i][1], "cluster": "NA", "result": "skipped", "time": "0"]) 109 } 110 111 markPassedTests() 112 } 113 114 void markPassedTests() { 115 echo "Marking passed tests in the tests map!" 116 117 withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { 118 sh """ 119 aws s3 ls "s3://percona-jenkins-artifactory/${JOB_NAME}/${env.GIT_SHORT_COMMIT}/" || : 120 """ 121 122 for (int i=0; i<tests.size(); i++) { 123 def testNameWithMysqlVersion = tests[i]["name"] +"-"+ tests[i]["mysql_ver"].replace(".", "-") 124 def file="${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testNameWithMysqlVersion" 125 def retFileExists = sh(script: "aws s3api head-object --bucket percona-jenkins-artifactory --key ${JOB_NAME}/${env.GIT_SHORT_COMMIT}/${file} >/dev/null 2>&1", returnStatus: true) 126 127 if (retFileExists == 0) { 128 tests[i]["result"] = "passed" 129 } 130 } 131 } 132 } 133 134 void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) { 135 sh """ 136 export KUBECONFIG=/tmp/$CLUSTER_NAME-$CLUSTER_SUFFIX 137 echo "========== KUBERNETES STATUS $LOCATION TEST ==========" 138 gcloud container clusters list|grep -E "NAME|$CLUSTER_NAME-$CLUSTER_SUFFIX " 139 echo 140 kubectl get nodes 141 echo 142 kubectl top nodes 143 echo 144 kubectl get pods --all-namespaces 145 echo 146 kubectl top pod --all-namespaces 147 echo 148 kubectl get events --field-selector type!=Normal --all-namespaces 149 echo "======================================================" 150 """ 151 } 152 153 TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |' 154 TestsReportXML = '<testsuite name=\\"PXC\\">\n' 155 156 void makeReport() { 157 def wholeTestAmount=tests.size() 158 def startedTestAmount = 0 159 160 for (int i=0; i<tests.size(); i++) { 161 def testResult = tests[i]["result"] 162 def testTime = tests[i]["time"] 163 def testNameWithMysqlVersion = tests[i]["name"] +"-"+ tests[i]["mysql_ver"].replace(".", "-") 164 def testUrl = "${testUrlPrefix}/${env.GIT_BRANCH}/${env.GIT_SHORT_COMMIT}/${testNameWithMysqlVersion}.log" 165 166 if (tests[i]["result"] != "skipped") { 167 startedTestAmount++ 168 } 169 TestsReport = TestsReport + "\r\n| "+ testNameWithMysqlVersion +" | ["+ tests[i]["result"] +"]("+ testUrl +") |" 170 TestsReportXML = TestsReportXML + '<testcase name=\\"' + testNameWithMysqlVersion + '\\" time=\\"' + testTime + '\\"><'+ testResult +'/></testcase>\n' 171 } 172 TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount|" 173 TestsReportXML = TestsReportXML + '</testsuite>\n' 174 } 175 176 void clusterRunner(String cluster) { 177 def clusterCreated=0 178 179 for (int i=0; i<tests.size(); i++) { 180 if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) { 181 tests[i]["result"] = "failure" 182 tests[i]["cluster"] = cluster 183 if (clusterCreated == 0) { 184 createCluster(cluster) 185 clusterCreated++ 186 } 187 runTest(i) 188 } 189 } 190 191 if (clusterCreated >= 1) { 192 shutdownCluster(cluster) 193 } 194 } 195 196 void runTest(Integer TEST_ID) { 197 def retryCount = 0 198 def testName = tests[TEST_ID]["name"] 199 def mysqlVer = tests[TEST_ID]["mysql_ver"] 200 def clusterSuffix = tests[TEST_ID]["cluster"] 201 def testNameWithMysqlVersion = "$testName-$mysqlVer".replace(".", "-") 202 203 waitUntil { 204 def timeStart = new Date().getTime() 205 try { 206 echo "The $testName-$mysqlVer test was started on cluster $CLUSTER_NAME-$clusterSuffix !" 207 tests[TEST_ID]["result"] = "failure" 208 209 timeout(time: 90, unit: 'MINUTES') { 210 sh """ 211 if [ $retryCount -eq 0 ]; then 212 export DEBUG_TESTS=0 213 else 214 export DEBUG_TESTS=1 215 fi 216 export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix 217 export MYSQL_VERSION=$mysqlVer 218 time bash e2e-tests/$testName/run 219 """ 220 } 221 pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testNameWithMysqlVersion") 222 tests[TEST_ID]["result"] = "passed" 223 return true 224 } 225 catch (exc) { 226 printKubernetesStatus("AFTER","$clusterSuffix") 227 if (retryCount >= 1 || currentBuild.nextBuild != null) { 228 currentBuild.result = 'FAILURE' 229 return true 230 } 231 retryCount++ 232 return false 233 } 234 finally { 235 def timeStop = new Date().getTime() 236 def durationSec = (timeStop - timeStart) / 1000 237 tests[TEST_ID]["time"] = durationSec 238 pushLogFile("$testNameWithMysqlVersion") 239 echo "The $testName-$mysqlVer test was finished!" 240 } 241 } 242 } 243 244 def skipBranchBuilds = true 245 if (env.CHANGE_URL) { 246 skipBranchBuilds = false 247 } 248 249 pipeline { 250 environment { 251 CLOUDSDK_CORE_DISABLE_PROMPTS = 1 252 CLEAN_NAMESPACE = 1 253 OPERATOR_NS = 'pxc-operator' 254 GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', , returnStdout: true).trim() 255 VERSION = "${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}" 256 CLUSTER_NAME = sh(script: "echo jen-pxc-${env.CHANGE_ID}-${GIT_SHORT_COMMIT}-${env.BUILD_NUMBER} | tr '[:upper:]' '[:lower:]'", , returnStdout: true).trim() 257 AUTHOR_NAME = sh(script: "echo ${CHANGE_AUTHOR_EMAIL} | awk -F'@' '{print \$1}'", , returnStdout: true).trim() 258 ENABLE_LOGGING = "true" 259 } 260 agent { 261 label 'docker' 262 } 263 options { 264 disableConcurrentBuilds(abortPrevious: true) 265 } 266 stages { 267 stage('Prepare') { 268 when { 269 expression { 270 !skipBranchBuilds 271 } 272 } 273 steps { 274 initTests() 275 script { 276 if (AUTHOR_NAME == 'null') { 277 AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", , returnStdout: true).trim() 278 } 279 for (comment in pullRequest.comments) { 280 println("Author: ${comment.user}, Comment: ${comment.body}") 281 if (comment.user.equals('JNKPercona')) { 282 println("delete comment") 283 comment.delete() 284 } 285 } 286 } 287 sh """ 288 sudo curl -s -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && sudo chmod +x /usr/local/bin/kubectl 289 kubectl version --client --output=yaml 290 291 curl -fsSL https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz | sudo tar -C /usr/local/bin --strip-components 1 -xzf - linux-amd64/helm 292 293 sudo sh -c "curl -s -L https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 > /usr/local/bin/yq" 294 sudo chmod +x /usr/local/bin/yq 295 296 sudo sh -c "curl -s -L https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 > /usr/local/bin/jq" 297 sudo chmod +x /usr/local/bin/jq 298 299 sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOF 300 [google-cloud-cli] 301 name=Google Cloud CLI 302 baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64 303 enabled=1 304 gpgcheck=1 305 repo_gpgcheck=0 306 gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg 307 EOF 308 sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin 309 310 curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense 311 312 sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true 313 sudo percona-release enable-only tools 314 sudo yum install -y percona-xtrabackup-80 | true 315 """ 316 317 withCredentials([file(credentialsId: 'cloud-secret-file', variable: 'CLOUD_SECRET_FILE')]) { 318 sh ''' 319 cp $CLOUD_SECRET_FILE e2e-tests/conf/cloud-secret.yml 320 ''' 321 } 322 deleteOldClusters("jen-pxc-$CHANGE_ID") 323 } 324 } 325 stage('Build docker image') { 326 when { 327 expression { 328 !skipBranchBuilds 329 } 330 } 331 steps { 332 withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { 333 sh ''' 334 DOCKER_TAG=perconalab/percona-xtradb-cluster-operator:$VERSION 335 docker_tag_file='./results/docker/TAG' 336 mkdir -p $(dirname ${docker_tag_file}) 337 echo ${DOCKER_TAG} > "${docker_tag_file}" 338 sg docker -c " 339 docker login -u '${USER}' -p '${PASS}' 340 export RELEASE=0 341 export IMAGE=\$DOCKER_TAG 342 docker buildx create --use 343 ./e2e-tests/build 344 docker logout 345 " 346 sudo rm -rf ./build 347 ''' 348 } 349 stash includes: 'results/docker/TAG', name: 'IMAGE' 350 archiveArtifacts 'results/docker/TAG' 351 } 352 } 353 stage('GoLicenseDetector test') { 354 when { 355 expression { 356 !skipBranchBuilds 357 } 358 } 359 steps { 360 sh """ 361 mkdir -p $WORKSPACE/src/github.com/percona 362 ln -s $WORKSPACE $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator 363 sg docker -c " 364 docker run \ 365 --rm \ 366 -v $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator:/go/src/github.com/percona/percona-xtradb-cluster-operator \ 367 -w /go/src/github.com/percona/percona-xtradb-cluster-operator \ 368 golang:1.21 sh -c ' 369 go install -mod=readonly github.com/google/go-licenses@latest; 370 /go/bin/go-licenses csv github.com/percona/percona-xtradb-cluster-operator/cmd/manager \ 371 | cut -d , -f 3 \ 372 | sort -u \ 373 > go-licenses-new || : 374 ' 375 " 376 diff -u e2e-tests/license/compare/go-licenses go-licenses-new 377 """ 378 } 379 } 380 stage('GoLicense test') { 381 when { 382 expression { 383 !skipBranchBuilds 384 } 385 } 386 steps { 387 sh ''' 388 mkdir -p $WORKSPACE/src/github.com/percona 389 ln -s $WORKSPACE $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator 390 sg docker -c " 391 docker run \ 392 --rm \ 393 -v $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator:/go/src/github.com/percona/percona-xtradb-cluster-operator \ 394 -w /go/src/github.com/percona/percona-xtradb-cluster-operator \ 395 -e GO111MODULE=on \ 396 -e GOFLAGS='-buildvcs=false' \ 397 golang:1.21 sh -c 'go build -v -o percona-xtradb-cluster-operator github.com/percona/percona-xtradb-cluster-operator/cmd/manager' 398 " 399 ''' 400 401 withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'GITHUB_TOKEN')]) { 402 sh """ 403 golicense -plain ./percona-xtradb-cluster-operator \ 404 | grep -v 'license not found' \ 405 | sed -r 's/^[^ ]+[ ]+//' \ 406 | sort \ 407 | uniq \ 408 > golicense-new || true 409 diff -u e2e-tests/license/compare/golicense golicense-new 410 """ 411 } 412 } 413 } 414 stage('Run tests for operator') { 415 when { 416 expression { 417 !skipBranchBuilds 418 } 419 } 420 options { 421 timeout(time: 4, unit: 'HOURS') 422 } 423 parallel { 424 stage('cluster1') { 425 steps { 426 clusterRunner('cluster1') 427 } 428 } 429 stage('cluster2') { 430 steps { 431 clusterRunner('cluster2') 432 } 433 } 434 stage('cluster3') { 435 steps { 436 clusterRunner('cluster3') 437 } 438 } 439 stage('cluster4') { 440 steps { 441 clusterRunner('cluster4') 442 } 443 } 444 stage('cluster5') { 445 steps { 446 clusterRunner('cluster5') 447 } 448 } 449 stage('cluster6') { 450 steps { 451 clusterRunner('cluster6') 452 } 453 } 454 stage('cluster7') { 455 steps { 456 clusterRunner('cluster7') 457 } 458 } 459 stage('cluster8') { 460 steps { 461 clusterRunner('cluster8') 462 } 463 } 464 stage('cluster9') { 465 steps { 466 clusterRunner('cluster9') 467 } 468 } 469 } 470 } 471 } 472 post { 473 always { 474 script { 475 echo "CLUSTER ASSIGNMENTS\n" + tests.toString().replace("], ","]\n").replace("]]","]").replaceFirst("\\[","") 476 477 if (currentBuild.result != null && currentBuild.result != 'SUCCESS' && currentBuild.nextBuild == null) { 478 try { 479 slackSend channel: "@${AUTHOR_NAME}", color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}" 480 } 481 catch (exc) { 482 slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}" 483 } 484 } 485 486 if (env.CHANGE_URL && currentBuild.nextBuild == null) { 487 for (comment in pullRequest.comments) { 488 println("Author: ${comment.user}, Comment: ${comment.body}") 489 if (comment.user.equals('JNKPercona')) { 490 println("delete comment") 491 comment.delete() 492 } 493 } 494 makeReport() 495 sh """ 496 echo "${TestsReportXML}" > TestsReport.xml 497 """ 498 step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0]) 499 archiveArtifacts '*.xml' 500 501 unstash 'IMAGE' 502 def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim() 503 TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n" 504 pullRequest.comment(TestsReport) 505 } 506 } 507 deleteOldClusters("$CLUSTER_NAME") 508 sh """ 509 sudo docker system prune --volumes -af 510 sudo rm -rf * 511 """ 512 deleteDir() 513 } 514 } 515 }