github.com/verrazzano/verrazzano@v1.7.1/ci/JenkinsfilePeriodicTests (about) 1 // Copyright (c) 2021, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 import groovy.transform.Field 5 6 @Field 7 def GIT_COMMIT_TO_USE = "" 8 @Field 9 def LAST_CLEAN_PERIODIC_COMMIT = "" 10 @Field 11 def LAST_PERIODIC_RUN_COMMIT = "" 12 @Field 13 def VERRAZZANO_DEV_VERSION = "" 14 @Field 15 def VERSIONED_IMAGES_FILENAME = "" 16 @Field 17 def TESTS_FAILED = false 18 @Field 19 def storeLocation="" 20 @Field 21 def verrazzanoPrefix="verrazzano-" 22 @Field 23 def fullBundle="" 24 @Field 25 def liteBundle="" 26 @Field 27 def SUSPECT_LIST = "" 28 @Field 29 def COMPARISON_URL_ON_FAILURE = "" 30 31 // The job name from which the verrazzano_images file is available to be copied to this job 32 // We will copy over and make it part of the artifacts of the periodic job, available when we want to release a candidate 33 @Field 34 def verrazzanoImagesJobProjectName = "verrazzano-examples" 35 @Field 36 def verrazzanoImagesFile = "verrazzano_images.txt" 37 @Field 38 def verrazzanoImagesBuildNumber = 0 // will be set to actual build number when the job is run 39 @Field 40 def periodicsUpToDate = false // If true, indicates that the periodics already passed at the latest commit 41 @Field 42 def periodicsUpToDateFailed = false // If true, indicates that the periodics already ran and failed at the latest commit 43 44 // Non Fields 45 // def branchSpecificSchedule = getCronSchedule() 46 47 // File containing the links to download the Verrazzano distributions 48 @Field 49 def verrazzanoDistributionsFile = "verrazzano_distributions.html" 50 51 def ociOsRegion = "us-phoenix-1" 52 def ociOsBucket = "$OCI_OS_BUCKET" 53 def urlTriggerBranchName = env.BRANCH_NAME.replace('/', '%2F') 54 def lastStableCommitFile = "last-stable-commit.txt" 55 56 pipeline { 57 options { 58 timeout(time: 12, unit: 'HOURS') 59 skipDefaultCheckout true 60 disableConcurrentBuilds() 61 timestamps () 62 } 63 64 agent { 65 docker { 66 image "${RUNNER_DOCKER_IMAGE}" 67 args "${RUNNER_DOCKER_ARGS}" 68 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 69 registryCredentialsId 'ocir-pull-and-push-account' 70 label "pipeline-job-large" 71 } 72 } 73 74 // triggers { 75 // URLTrigger( 76 // cronTabSpec: branchSpecificSchedule, 77 // entries: [ 78 // URLTriggerEntry( 79 // url: "https://objectstorage.${ociOsRegion}.oraclecloud.com/n/${OS_NAMESPACE_URL_TRIGGER}/b/${ociOsBucket}/o/${urlTriggerBranchName}/${lastStableCommitFile}", 80 // checkETag: false, 81 // checkStatus: true, 82 // statusCode: 403, 83 // checkLastModificationDate: true, 84 // timeout: 200, 85 // requestHeaders: [ 86 // RequestHeader( headerName: "Accept" , headerValue: "application/json" ) 87 // ], 88 // contentTypes: [ 89 // MD5Sum() 90 // ] 91 // ) 92 // ] 93 // ) 94 // } 95 96 parameters { 97 string (name: 'TAGGED_TESTS', 98 defaultValue: '', 99 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 100 trim: true) 101 string (name: 'INCLUDED_TESTS', 102 defaultValue: '.*', 103 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 104 trim: true) 105 string (name: 'EXCLUDED_TESTS', 106 defaultValue: '_excluded_test', 107 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 108 trim: true) 109 booleanParam (description: 'Force execution of the tests even if up-to-date', name: 'FORCE', defaultValue: false) 110 booleanParam (description: 'Skip test execution (for debugging)', name: 'DRY_RUN', defaultValue: false) 111 } 112 113 environment { 114 IS_PERIODIC_PIPELINE = "true" 115 OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment') 116 OCIR_SCAN_TARGET = credentials('ocir-scan-target') 117 OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry') 118 OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path') 119 DOCKER_SCAN_CREDS = credentials('v8odev-ocir') 120 DOCKER_CREDS = credentials('github-packages-credentials-rw') 121 DOCKER_REPO = 'ghcr.io' 122 123 OCI_CLI_AUTH="instance_principal" 124 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 125 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 126 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 127 128 STABLE_COMMIT_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-stable-commit.txt" 129 LAST_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-periodic-run-commit.txt" 130 CLEAN_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test/verrazzano_periodic-commit.txt" 131 132 STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt" 133 LAST_PERIODIC_LOCATION = "${WORKSPACE}/last-periodic-run-commit.txt" 134 CLEAN_PERIODIC_LOCATION = "${WORKSPACE}/last-clean-periodic-commit.txt" 135 136 OCI_OS_REGION="us-phoenix-1" 137 138 PIPELINE_OWNERS = credentials('periodictests-owners') 139 } 140 141 // This job runs against the latest stable master commit. That is defined as the last clean master build and test run whose 142 // commit has been stored in object storage. This job will fetch that commit from master and run extended tests using that. 143 // This job is NOT currently setup to run extended tests from other branches, if you need to run those extended jobs you will 144 // need to run those against your branch individually. 145 146 stages { 147 stage('Check last clean periodic') { 148 steps { 149 sh """ 150 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${STABLE_COMMIT_OS_LOCATION} --file ${STABLE_COMMIT_LOCATION} 151 """ 152 153 script { 154 // Check if there is already a clean periodic run at this commit already, and set the display name if 155 // it already is tested, or if doing a special run type (dry run, etc...) 156 preliminaryChecks() 157 } 158 } 159 } 160 stage('Clean workspace and checkout') { 161 when { 162 allOf { 163 expression { return runTests() } 164 } 165 } 166 steps { 167 script { 168 cleanWorkspaceAndCheckout() 169 } 170 } 171 } 172 stage ('Periodic Test Suites') { 173 when { 174 allOf { 175 expression { return runTests() } 176 } 177 } 178 parallel { 179 stage('OCI DNS tests with instance principal') { 180 steps { 181 retry(count: JOB_PROMOTION_RETRIES) { 182 script { 183 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 184 parameters: [ 185 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 186 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 187 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 188 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 189 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 190 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 191 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 192 ], wait: true 193 } 194 } 195 } 196 } 197 stage('Verrazzano in OCNE environment') { 198 steps { 199 retry(count: JOB_PROMOTION_RETRIES) { 200 script { 201 build job: "/verrazzano-ocne/${CLEAN_BRANCH_NAME}", 202 parameters: [ 203 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 204 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 205 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 206 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 207 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 208 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 209 ], wait: true 210 } 211 } 212 } 213 } 214 stage('Backup and Restore Tests') { 215 steps { 216 retry(count: JOB_PROMOTION_RETRIES) { 217 script { 218 build job: "/verrazzano-backup-all-test-oke/${CLEAN_BRANCH_NAME}", 219 parameters: [ 220 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 221 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: 'NONE'), 222 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 223 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 224 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 225 ], wait: true 226 } 227 } 228 } 229 } 230 stage('Kind Acceptance Tests With HA and Grafana DB on 1.27') { 231 steps { 232 retry(count: JOB_PROMOTION_RETRIES) { 233 script { 234 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 235 parameters: [ 236 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 237 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 238 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 239 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 240 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 241 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 242 booleanParam(name: 'INSTALL_HA', value: true), 243 booleanParam(name: 'USE_DB_FOR_GRAFANA', value: true) 244 ], wait: true 245 } 246 } 247 } 248 } 249 stage('Kind Acceptance Tests on 1.24') { 250 steps { 251 retry(count: JOB_PROMOTION_RETRIES) { 252 script { 253 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 254 parameters: [ 255 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.24'), 256 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 257 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 258 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 259 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 260 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 261 booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: true) 262 ], wait: true 263 } 264 } 265 } 266 } 267 stage('Kind Acceptance Tests on 1.25') { 268 steps { 269 retry(count: JOB_PROMOTION_RETRIES) { 270 script { 271 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 272 parameters: [ 273 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.25'), 274 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 275 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 276 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 277 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 278 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 279 booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: true) 280 ], wait: true 281 } 282 } 283 } 284 } 285 stage('Kind Acceptance Tests on 1.26') { 286 steps { 287 retry(count: JOB_PROMOTION_RETRIES) { 288 script { 289 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 290 parameters: [ 291 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.26'), 292 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 293 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 294 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 295 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 296 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 297 booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: true) 298 ], wait: true 299 } 300 } 301 } 302 } 303 stage('HA tests') { 304 steps { 305 retry(count: JOB_PROMOTION_RETRIES) { 306 script { 307 build job: "/verrazzano-ha-tests/${CLEAN_BRANCH_NAME}", 308 parameters: [ 309 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 310 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 311 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 312 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 313 ], wait: false 314 } 315 } 316 } 317 } 318 stage('Upgrade Resiliency tests') { 319 steps { 320 script { 321 build job: "/verrazzano-upgrade-resiliency-tests/${CLEAN_BRANCH_NAME}", 322 parameters: [ 323 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 324 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 325 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 326 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 327 ], wait: true 328 } 329 } 330 } 331 stage('Uninstall Resiliency tests') { 332 steps { 333 script { 334 build job: "/verrazzano-uninstall-resiliency-suite/${CLEAN_BRANCH_NAME}", 335 parameters: [ 336 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 337 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 338 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 339 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 340 ], wait: true 341 } 342 } 343 } 344 stage('Kind Acceptance Tests on 1.27 Non-Calico') { 345 steps { 346 retry(count: JOB_PROMOTION_RETRIES) { 347 script { 348 // Do not use Calico 349 build job: "verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 350 parameters: [ 351 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 352 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 353 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 354 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 355 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 356 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 357 ], wait: true 358 } 359 } 360 } 361 post { 362 failure { 363 script { 364 TESTS_FAILED = true 365 } 366 } 367 } 368 } 369 stage('Kind Acceptance Tests on 1.27 sslip.io') { 370 steps { 371 retry(count: JOB_PROMOTION_RETRIES) { 372 script { 373 // test with sslip.io 374 build job: "verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 375 parameters: [ 376 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 377 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 378 string(name: 'WILDCARD_DNS_DOMAIN', value: 'sslip.io'), 379 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 380 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 381 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 382 ], wait: true 383 } 384 } 385 } 386 post { 387 failure { 388 script { 389 TESTS_FAILED = true 390 } 391 } 392 } 393 } 394 stage('Dynamic Configuration Tests') { 395 steps { 396 retry(count: JOB_PROMOTION_RETRIES) { 397 script { 398 build job: "verrazzano-dynamic-config-suite/${CLEAN_BRANCH_NAME}", 399 parameters: [ 400 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 401 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 402 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 403 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 404 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 405 ], wait: true 406 } 407 } 408 } 409 post { 410 failure { 411 script { 412 TESTS_FAILED = true 413 } 414 } 415 } 416 } 417 stage('OCI DNS/ACME-Staging Tests') { 418 steps { 419 retry(count: JOB_PROMOTION_RETRIES) { 420 script { 421 build job: "verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 422 parameters: [ 423 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 424 string(name: 'CERT_ISSUER', value: "acme"), 425 string(name: 'ACME_ENVIRONMENT', value: "staging"), 426 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 427 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 428 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 429 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 430 ], wait: true 431 } 432 } 433 } 434 post { 435 failure { 436 script { 437 TESTS_FAILED = true 438 } 439 } 440 } 441 } 442 stage('OCI Service Integration Tests') { 443 steps { 444 retry(count: JOB_PROMOTION_RETRIES) { 445 script { 446 build job: "/verrazzano-oci-integration-tests/${CLEAN_BRANCH_NAME}", 447 parameters: [ 448 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 449 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 450 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 451 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 452 ], wait: true 453 } 454 } 455 } 456 post { 457 failure { 458 script { 459 TESTS_FAILED = true 460 } 461 } 462 } 463 } 464 stage('Verrazzano Examples') { 465 steps { 466 retry(count: JOB_PROMOTION_RETRIES) { 467 script { 468 def builtExamples = build job: "/verrazzano-examples/${CLEAN_BRANCH_NAME}", 469 parameters: [ 470 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 471 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 472 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 473 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 474 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 475 ], wait: true 476 // The verrazzano-examples job is the one from which we get the verrazzano images file 477 if (builtExamples.currentResult == 'SUCCESS') { 478 verrazzanoImagesBuildNumber = builtExamples.number 479 copyImagesFromExamplesBuild() 480 } 481 } 482 } 483 } 484 post { 485 failure { 486 script { 487 TESTS_FAILED = true 488 } 489 } 490 } 491 } 492 stage('Verrazzano Distribution and Private Registry') { 493 steps { 494 script { 495 build job: "/verrazzano-distributions-testing/${CLEAN_BRANCH_NAME}", 496 parameters: [ 497 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 498 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 499 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 500 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 501 string(name: 'RUN_TESTS', value: env.IS_PERIODIC_PIPELINE), 502 ], wait: true 503 } 504 } 505 } 506 stage('ClusterAPI cluster creation tests using KIND as admin') { 507 steps { 508 retry(count: JOB_PROMOTION_RETRIES) { 509 script { 510 build job: "verrazzano-clusterAPI-kind-tests/${CLEAN_BRANCH_NAME}", 511 parameters: [ 512 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 513 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 514 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 515 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 516 ], wait: false 517 } 518 } 519 } 520 post { 521 failure { 522 script { 523 TESTS_FAILED = true 524 } 525 } 526 } 527 } 528 stage('OCNE Cluster Driver') { 529 steps { 530 retry(count: JOB_PROMOTION_RETRIES) { 531 script { 532 build job: "verrazzano-ocne-cluster-driver-tests/${CLEAN_BRANCH_NAME}", 533 parameters: [ 534 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 535 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 536 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 537 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 538 ], wait: true 539 } 540 } 541 } 542 post { 543 failure { 544 script { 545 TESTS_FAILED = true 546 } 547 } 548 } 549 } 550 stage('OKE CAPI Cluster Driver') { 551 steps { 552 retry(count: JOB_PROMOTION_RETRIES) { 553 script { 554 build job: "verrazzano-oke-capi-cluster-driver-tests/${CLEAN_BRANCH_NAME}", 555 parameters: [ 556 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 557 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 558 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 559 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 560 ], wait: true 561 } 562 } 563 } 564 post { 565 failure { 566 script { 567 TESTS_FAILED = true 568 } 569 } 570 } 571 } 572 } 573 } 574 stage('Release Candidate Validation Checks') { 575 when { 576 allOf { 577 expression { return runTests() } 578 } 579 } 580 steps { 581 script { 582 releaseValidationChecks() 583 } 584 } 585 } 586 stage('Update Last Clean Periodic Test') { 587 environment { 588 GIT_COMMIT_USED = "${env.GIT_COMMIT}" 589 } 590 when { 591 allOf { 592 expression { return runTests() } 593 } 594 } 595 steps { 596 script { 597 sh """ 598 ci/scripts/update_last_clean_periodic_test.sh ${VERRAZZANO_DEV_VERSION} ${SHORT_COMMIT_HASH} 599 """ 600 } 601 } 602 post { 603 success { 604 script { 605 captureDistributionURLs() 606 } 607 } 608 } 609 } 610 } 611 post { 612 always { 613 script { 614 // We don't update any object storage artifacts if this run didn't re-run anything (ie: if it was "UpToDate") 615 if (!periodicsUpToDate && !periodicsUpToDateFailed) { 616 sh """ 617 # Update the last periodic commit 618 echo "git-commit=${env.GIT_COMMIT}" > commit-used.txt 619 cat commit-used.txt 620 oci --region ${OCI_OS_REGION} os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${LAST_PERIODIC_OS_LOCATION} --file commit-used.txt 621 """ 622 } else { 623 println("Skipping update of the last periodic commit used. This run was marked UP-TO-DATE so it is already there") 624 } 625 } 626 archiveArtifacts artifacts: "**/prerelease_validation.out,**/release_status.out,**/${verrazzanoImagesFile},**/${verrazzanoDistributionsFile}", allowEmptyArchive: true 627 } 628 failure { 629 script { 630 failedOrAborted() 631 } 632 } 633 aborted { 634 script { 635 failedOrAborted() 636 } 637 } 638 cleanup { 639 deleteDir() 640 } 641 } 642 } 643 644 def copyImagesFromExamplesBuild() { 645 script { 646 if (verrazzanoImagesBuildNumber > 0) { 647 copyArtifacts(projectName: "${verrazzanoImagesJobProjectName}/${CLEAN_BRANCH_NAME}", 648 selector: specific("${verrazzanoImagesBuildNumber}"), 649 filter: verrazzanoImagesFile) 650 sh """ 651 # Save the images.txt into the commit specific location, if we have a clean periodic it will copy the images from here to the branch as the current ones 652 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VERSIONED_IMAGES_FILENAME} --file ${verrazzanoImagesFile} 653 """ 654 } else { 655 println("Cannot copy ${verrazzanoImagesFile} from the project ${verrazzanoImagesJobProjectName} - no build number is available to copy from.") 656 } 657 } 658 } 659 660 def failedOrAborted() { 661 if (isAlertingEnabled()) { 662 if (isPagerDutyEnabled()) { 663 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", 664 incDescription: "Verrazzano Periodic Tests: ${env.JOB_NAME} - Failed", 665 incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 666 } 667 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST} ${PIPELINE_OWNERS}\n\nChange comparison: ${COMPARISON_URL_ON_FAILURE}" ) 668 echo "done alerts" 669 } 670 } 671 672 // Preliminary job checks and display updates 673 def preliminaryChecks() { 674 // Get the last stable commit ID to pass the triggered tests 675 def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}" 676 GIT_COMMIT_TO_USE = stableCommitProps['git-commit'] 677 echo "Last stable commit: ${GIT_COMMIT_TO_USE}" 678 679 LAST_CLEAN_PERIODIC_COMMIT=getLastCleanPeriodicCommit() 680 echo "Last clean periodics commit: ${LAST_CLEAN_PERIODIC_COMMIT}" 681 682 if (LAST_CLEAN_PERIODIC_COMMIT == GIT_COMMIT_TO_USE) { 683 // If we had a clean periodic run and the commit hasn't changed 684 periodicsUpToDate = true 685 } else { 686 // Check if we are still at the same commit previously run (if so we know it wasn't clean and it failed in some way) 687 LAST_PERIODIC_RUN_COMMIT=getLastPeriodicRunCommit() 688 if (LAST_PERIODIC_RUN_COMMIT != null && LAST_PERIODIC_RUN_COMMIT == GIT_COMMIT_TO_USE) { 689 periodicsUpToDateFailed = true 690 } 691 } 692 693 echo "Up to date: ${periodicsUpToDate}" 694 echo "Dry run: ${params.DRY_RUN}" 695 echo "Force run: ${params.FORCE}" 696 echo "Execute tests: " + runTests() 697 698 // Indicate in title if run is up-to-date or dry-run 699 if (params.DRY_RUN) { 700 currentBuild.displayName = "${currentBuild.displayName} : DRY-RUN" 701 } 702 if (periodicsUpToDate) { 703 currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE" 704 } 705 706 if (params.FORCE) { 707 currentBuild.displayName = "${currentBuild.displayName} : FORCE" 708 } else if (periodicsUpToDateFailed) { 709 currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE-FAILED" 710 currentBuild.result = 'FAILURE' 711 error('Failing the build since the current commit matches the commit of previously failing periodic build') 712 } 713 714 if (runTests()) { 715 echo "Executing periodic tests for commit ${GIT_COMMIT_TO_USE}" 716 } 717 } 718 719 def dockerLogins() { 720 try { 721 sh """ 722 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 723 """ 724 } catch(error) { 725 echo "docker login failed, retrying after sleep" 726 retry(4) { 727 sleep(30) 728 sh """ 729 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 730 """ 731 } 732 } 733 if (!(env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) { 734 try { 735 sh """ 736 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 737 """ 738 } catch(error) { 739 echo "docker login failed, retrying after sleep" 740 retry(4) { 741 sleep(30) 742 sh """ 743 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 744 """ 745 } 746 } 747 } 748 } 749 750 def scmCheckout() { 751 echo "${NODE_LABELS}" 752 echo "SCM checkout of ${GIT_COMMIT_TO_USE}" 753 def scmInfo = checkout([ 754 $class: 'GitSCM', 755 branches: [[name: GIT_COMMIT_TO_USE]], 756 doGenerateSubmoduleConfigurations: false, 757 extensions: [], 758 submoduleCfg: [], 759 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 760 env.GIT_COMMIT = scmInfo.GIT_COMMIT 761 env.GIT_BRANCH = scmInfo.GIT_BRANCH 762 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 763 // If the commit we were handed is not what the SCM says we are using, fail 764 if (!env.GIT_COMMIT.equals(GIT_COMMIT_TO_USE)) { 765 error( "SCM didn't checkout the commit we expected. Expected: ${GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}") 766 } 767 768 if (LAST_CLEAN_PERIODIC_COMMIT != null) { 769 COMPARISON_URL_ON_FAILURE = "https://github.com/verrazzano/verrazzano/compare/${LAST_CLEAN_PERIODIC_COMMIT}...${GIT_COMMIT_TO_USE}" 770 def lastClean = "${LAST_CLEAN_PERIODIC_COMMIT}" 771 def currentStable = "${GIT_COMMIT_TO_USE}" 772 def commitList = getCommitListFromGitLog(lastClean, currentStable) 773 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 774 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 775 SUSPECT_LIST = getSuspectList(commitList, userMappings) 776 echo "Suspect list: ${SUSPECT_LIST}" 777 } 778 } 779 echo "URL if fails: ${COMPARISON_URL_ON_FAILURE}" 780 } 781 782 def cleanWorkspaceAndCheckout() { 783 scmCheckout() 784 dockerLogins() 785 def props = readProperties file: '.verrazzano-development-version' 786 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 787 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 788 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 789 // update the description with some meaningful info 790 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + GIT_COMMIT_TO_USE 791 storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 792 fullBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 793 liteBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 794 VERSIONED_IMAGES_FILENAME = "verrazzano_${VERRAZZANO_DEV_VERSION}-images.txt" 795 } 796 797 // Returns the last clean commit for the periodics, or null if the commit file does not exist yet. 798 // - fails the pipeline if any error other than 404 is returned by the OCI CLI 799 def getLastCleanPeriodicCommit() { 800 lastPeriodicCommitCommandOutput = sh ( 801 label: "Get last clean periodic commit ID", 802 script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_PERIODIC_OS_LOCATION} --file ${CLEAN_PERIODIC_LOCATION} 2>&1 || true", 803 returnStdout: true 804 ).trim() 805 echo "command out: ${lastPeriodicCommitCommandOutput}" 806 if (lastPeriodicCommitCommandOutput.length() > 0) { 807 // We can get warning messages here as well even when the command succeeded, so be more precise on the checking 808 if (lastPeriodicCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) { 809 // If we think we had a status: NNN, we ignore 404 and fail for others 810 assert lastPeriodicCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 811 } else { 812 // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error 813 sh """ 814 if [ ! -f ${CLEAN_PERIODIC_LOCATION} ]; then 815 echo "An unexpected error occurred getting last clean periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 816 exit 1 817 fi 818 """ 819 } 820 } 821 // Get the commit ID for the last known clean pass of the Periodic tests 822 def cleanPeriodicsCommitProps = readProperties file: "${CLEAN_PERIODIC_LOCATION}" 823 return cleanPeriodicsCommitProps['git-commit'] 824 } 825 826 // Returns the last run commit for the periodics, or null if the commit file does not exist yet. 827 // - fails the pipeline if any error other than 404 is returned by the OCI CLI 828 def getLastPeriodicRunCommit() { 829 lastPeriodicCommitCommandOutput = sh ( 830 label: "Get last clean periodic commit ID", 831 script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${LAST_PERIODIC_OS_LOCATION} --file ${LAST_PERIODIC_LOCATION} 2>&1 || true", 832 returnStdout: true 833 ).trim() 834 echo "command out: ${lastPeriodicCommitCommandOutput}" 835 if (lastPeriodicCommitCommandOutput.length() > 0) { 836 // We can get warning messages here as well even when the command succeeded, so be more precise on the checking 837 if (lastPeriodicCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) { 838 // If we think we had a status: NNN, we ignore 404 and fail for others 839 assert lastPeriodicCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 840 } else { 841 // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error 842 sh """ 843 if [ ! -f ${LAST_PERIODIC_LOCATION} ]; then 844 echo "An unexpected error occurred getting last periodic run commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 845 exit 1 846 fi 847 """ 848 } 849 } 850 // Get the commit ID for the last known clean pass of the Periodic tests 851 def lastPeriodicsCommitProps = readProperties file: "${LAST_PERIODIC_LOCATION}" 852 return lastPeriodicsCommitProps['git-commit'] 853 } 854 855 // Checks all the conditions gating test execution and collates the result 856 def runTests() { 857 return params.FORCE || ( ! periodicsUpToDate && ! periodicsUpToDateFailed && ! params.DRY_RUN ) 858 } 859 860 def isAlertingEnabled() { 861 // this controls whether any alerting happens for these tests 862 if (NOTIFY_PERIODIC_FAILURES.equals("true") && (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) { 863 echo "Alert notifications enabled for ${env.BRANCH_NAME}" 864 return true 865 } 866 return false 867 } 868 869 def releaseValidationChecks() { 870 def built = build job: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 871 parameters: [ 872 string(name: 'COMMIT_TO_USE', value: env.GIT_COMMIT) 873 ], wait: true, propagate: false 874 println("Result of verrazzano-prerelease-check is ${built.result}") 875 dir ("${WORKSPACE}") { 876 copyArtifacts(projectName: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 877 selector: specific("${built.number}")); 878 def releaseStatus = readFile file: "release_status.out" 879 currentBuild.displayName = "${currentBuild.displayName} : ${releaseStatus}" 880 } 881 } 882 883 def isPagerDutyEnabled() { 884 // this additionally controls whether PD alerts are enabled (note that you must also enable alerting in general as well if you want these) 885 if (NOTIFY_PAGERDUTY_PERIODIC_FAILURES.equals("true")) { 886 echo "Pager-Duty notifications enabled via global override setting" 887 return true 888 } 889 return false 890 } 891 892 // def getCronSchedule() { 893 // if (env.BRANCH_NAME.equals("master")) { 894 // return "H */6 * * *" 895 // } else if (env.BRANCH_NAME.startsWith("release-")) { 896 // return "@daily" 897 // } 898 // return "" 899 // } 900 901 // Called in Stage Clean workspace and checkout steps 902 def getCommitListFromGitLog(lastClean, currentStable) { 903 echo "Checking for change sets" 904 def commitList = sh(returnStdout: true, script: "git log ${lastClean}...${currentStable} --oneline | cut -d \" \" -f 1").trim().split('\n') 905 for (int i = 0; i < commitList.size(); i++) { 906 echo "Found commit id: ${commitList[i]}" 907 } 908 return commitList 909 } 910 911 def trimIfGithubNoreplyUser(userIn) { 912 if (userIn == null) { 913 echo "Not a github noreply user, not trimming: ${userIn}" 914 return userIn 915 } 916 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 917 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 918 return userOut; 919 } 920 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 921 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 922 return userOut; 923 } 924 if (userIn.matches(".*@users.noreply.github.com")) { 925 def userOut = userIn.substring(0, userIn.indexOf("@")) 926 return userOut; 927 } 928 echo "Not a github noreply user, not trimming: ${userIn}" 929 return userIn 930 } 931 932 def getSuspectList(commitList, userMappings) { 933 def retValue = "" 934 def suspectList = [] 935 if (commitList == null || commitList.size() == 0) { 936 echo "No commits to form suspect list" 937 } else { 938 for (int i = 0; i < commitList.size(); i++) { 939 def id = commitList[i] 940 try { 941 def gitAuthor = sh( 942 script: "git log --format='%ae' '$id^!'", 943 returnStdout: true 944 ).trim() 945 if (gitAuthor != null) { 946 def author = trimIfGithubNoreplyUser(gitAuthor) 947 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 948 if (userMappings.containsKey(author)) { 949 def slackUser = userMappings.get(author) 950 if (!suspectList.contains(slackUser)) { 951 echo "Added ${slackUser} as suspect" 952 retValue += " ${slackUser}" 953 suspectList.add(slackUser) 954 } 955 } else { 956 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 957 if (!suspectList.contains(author)) { 958 echo "Added ${author} as suspect" 959 retValue += " ${author}" 960 suspectList.add(author) 961 } 962 } 963 } else { 964 echo "No author returned from git" 965 } 966 } catch (Exception e) { 967 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 968 } 969 } 970 } 971 def startedByUser = ""; 972 def causes = currentBuild.getBuildCauses() 973 echo "causes: " + causes.toString() 974 for (cause in causes) { 975 def causeString = cause.toString() 976 echo "current cause: " + causeString 977 def causeInfo = readJSON text: causeString 978 if (causeInfo.userId != null) { 979 startedByUser = causeInfo.userId 980 } 981 } 982 983 if (startedByUser.length() > 0) { 984 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 985 def author = trimIfGithubNoreplyUser(startedByUser) 986 echo "DEBUG: author: ${startedByUser}, ${author}" 987 if (userMappings.containsKey(author)) { 988 def slackUser = userMappings.get(author) 989 if (!suspectList.contains(slackUser)) { 990 echo "Added ${slackUser} as suspect" 991 retValue += " ${slackUser}" 992 suspectList.add(slackUser) 993 } 994 } else { 995 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 996 if (!suspectList.contains(author)) { 997 echo "Added ${author} as suspect" 998 retValue += " ${author}" 999 suspectList.add(author) 1000 } 1001 } 1002 } else { 1003 echo "Build not started by a user, not adding to notification list" 1004 } 1005 echo "returning suspect list: ${retValue}" 1006 return retValue 1007 } 1008 1009 @NonCPS 1010 List extractReleaseTags(final String fileContent) { 1011 List releases = [] 1012 fileContent.eachLine { tag -> 1013 releases << tag 1014 } 1015 return releases 1016 } 1017 1018 def getLatestReleaseVersion() { 1019 final String releaseTags = readFile(file: "${workspace}/tags.txt") 1020 list gitTags = extractReleaseTags(releaseTags) 1021 echo "gitTags = ${gitTags}" 1022 return gitTags.pop() 1023 } 1024 1025 // Create a html file containing the links to the Verrazzano distributions 1026 def captureDistributionURLs() { 1027 script { 1028 def BRANCH_IN_OS = "${env.BRANCH_NAME.replace("/", "%252F")}" 1029 BASE_URL="https://objectstorage.${OCI_OS_REGION}.oraclecloud.com/n/${OCI_OS_NAMESPACE}/b/${OCI_OS_BUCKET}/o/${BRANCH_IN_OS}-last-clean-periodic-test" 1030 LITE_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 1031 FULL_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 1032 sh """ 1033 cat <<EOF > ${WORKSPACE}/${verrazzanoDistributionsFile} 1034 <!DOCTYPE html> 1035 <html> 1036 <body> 1037 <b>Verrazzano Release Distributions</b> 1038 <ul> 1039 <li><a href="${LITE_BUNDLE}">Verrazzano Lite Distribution</a></li> 1040 <li><a href="${FULL_BUNDLE}">Verrazzano Full Distribution</a></li> 1041 </ul> 1042 </body> 1043 </html> 1044 EOF 1045 """ 1046 } 1047 }