github.com/verrazzano/verrazzano@v1.7.0/ci/JenkinsfilePeriodicTests (about) 1 // Copyright (c) 2021, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 import groovy.transform.Field 5 6 @Field 7 def GIT_COMMIT_TO_USE = "" 8 @Field 9 def LAST_CLEAN_PERIODIC_COMMIT = "" 10 @Field 11 def LAST_PERIODIC_RUN_COMMIT = "" 12 @Field 13 def VERRAZZANO_DEV_VERSION = "" 14 @Field 15 def VERSIONED_IMAGES_FILENAME = "" 16 @Field 17 def TESTS_FAILED = false 18 @Field 19 def storeLocation="" 20 @Field 21 def verrazzanoPrefix="verrazzano-" 22 @Field 23 def fullBundle="" 24 @Field 25 def liteBundle="" 26 @Field 27 def SUSPECT_LIST = "" 28 @Field 29 def COMPARISON_URL_ON_FAILURE = "" 30 31 // The job name from which the verrazzano_images file is available to be copied to this job 32 // We will copy over and make it part of the artifacts of the periodic job, available when we want to release a candidate 33 @Field 34 def verrazzanoImagesJobProjectName = "verrazzano-examples" 35 @Field 36 def verrazzanoImagesFile = "verrazzano_images.txt" 37 @Field 38 def verrazzanoImagesBuildNumber = 0 // will be set to actual build number when the job is run 39 @Field 40 def periodicsUpToDate = false // If true, indicates that the periodics already passed at the latest commit 41 @Field 42 def periodicsUpToDateFailed = false // If true, indicates that the periodics already ran and failed at the latest commit 43 44 // Non Fields 45 // def branchSpecificSchedule = getCronSchedule() 46 47 // File containing the links to download the Verrazzano distributions 48 @Field 49 def verrazzanoDistributionsFile = "verrazzano_distributions.html" 50 51 def ociOsRegion = "us-phoenix-1" 52 def ociOsBucket = "verrazzano-builds" 53 def urlTriggerBranchName = env.BRANCH_NAME.replace('/', '%2F') 54 def lastStableCommitFile = "last-stable-commit.txt" 55 56 pipeline { 57 options { 58 timeout(time: 12, unit: 'HOURS') 59 skipDefaultCheckout true 60 disableConcurrentBuilds() 61 timestamps () 62 } 63 64 agent { 65 docker { 66 image "${RUNNER_DOCKER_IMAGE}" 67 args "${RUNNER_DOCKER_ARGS}" 68 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 69 registryCredentialsId 'ocir-pull-and-push-account' 70 label "pipeline-job-large" 71 } 72 } 73 74 // triggers { 75 // URLTrigger( 76 // cronTabSpec: branchSpecificSchedule, 77 // entries: [ 78 // URLTriggerEntry( 79 // url: "https://objectstorage.${ociOsRegion}.oraclecloud.com/n/${OS_NAMESPACE_URL_TRIGGER}/b/${ociOsBucket}/o/${urlTriggerBranchName}/${lastStableCommitFile}", 80 // checkETag: false, 81 // checkStatus: true, 82 // statusCode: 403, 83 // checkLastModificationDate: true, 84 // timeout: 200, 85 // requestHeaders: [ 86 // RequestHeader( headerName: "Accept" , headerValue: "application/json" ) 87 // ], 88 // contentTypes: [ 89 // MD5Sum() 90 // ] 91 // ) 92 // ] 93 // ) 94 // } 95 96 parameters { 97 string (name: 'TAGGED_TESTS', 98 defaultValue: '', 99 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 100 trim: true) 101 string (name: 'INCLUDED_TESTS', 102 defaultValue: '.*', 103 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 104 trim: true) 105 string (name: 'EXCLUDED_TESTS', 106 defaultValue: '_excluded_test', 107 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 108 trim: true) 109 booleanParam (description: 'Force execution of the tests even if up-to-date', name: 'FORCE', defaultValue: false) 110 booleanParam (description: 'Skip test execution (for debugging)', name: 'DRY_RUN', defaultValue: false) 111 } 112 113 environment { 114 IS_PERIODIC_PIPELINE = "true" 115 OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment') 116 OCIR_SCAN_TARGET = credentials('ocir-scan-target') 117 OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry') 118 OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path') 119 DOCKER_SCAN_CREDS = credentials('v8odev-ocir') 120 DOCKER_CREDS = credentials('github-packages-credentials-rw') 121 DOCKER_REPO = 'ghcr.io' 122 123 OCI_CLI_AUTH="instance_principal" 124 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 125 OCI_OS_BUCKET="verrazzano-builds" 126 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 127 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 128 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 129 130 STABLE_COMMIT_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-stable-commit.txt" 131 LAST_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-periodic-run-commit.txt" 132 CLEAN_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test/verrazzano_periodic-commit.txt" 133 134 STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt" 135 LAST_PERIODIC_LOCATION = "${WORKSPACE}/last-periodic-run-commit.txt" 136 CLEAN_PERIODIC_LOCATION = "${WORKSPACE}/last-clean-periodic-commit.txt" 137 138 OCI_OS_REGION="us-phoenix-1" 139 140 PIPELINE_OWNERS = credentials('periodictests-owners') 141 } 142 143 // This job runs against the latest stable master commit. That is defined as the last clean master build and test run whose 144 // commit has been stored in object storage. This job will fetch that commit from master and run extended tests using that. 145 // This job is NOT currently setup to run extended tests from other branches, if you need to run those extended jobs you will 146 // need to run those against your branch individually. 147 148 stages { 149 stage('Check last clean periodic') { 150 steps { 151 sh """ 152 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${STABLE_COMMIT_OS_LOCATION} --file ${STABLE_COMMIT_LOCATION} 153 """ 154 155 script { 156 // Check if there is already a clean periodic run at this commit already, and set the display name if 157 // it already is tested, or if doing a special run type (dry run, etc...) 158 preliminaryChecks() 159 } 160 } 161 } 162 stage('Clean workspace and checkout') { 163 when { 164 allOf { 165 expression { return runTests() } 166 } 167 } 168 steps { 169 script { 170 cleanWorkspaceAndCheckout() 171 } 172 } 173 } 174 stage ('Periodic Test Suites') { 175 when { 176 allOf { 177 expression { return runTests() } 178 } 179 } 180 parallel { 181 stage('OCI DNS tests with instance principal') { 182 steps { 183 retry(count: JOB_PROMOTION_RETRIES) { 184 script { 185 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 186 parameters: [ 187 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 188 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 189 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 190 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 191 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 192 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 193 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 194 ], wait: true 195 } 196 } 197 } 198 } 199 stage('Verrazzano in OCNE environment') { 200 steps { 201 retry(count: JOB_PROMOTION_RETRIES) { 202 script { 203 build job: "/verrazzano-ocne/${CLEAN_BRANCH_NAME}", 204 parameters: [ 205 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 206 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 207 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 208 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 209 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 210 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 211 ], wait: true 212 } 213 } 214 } 215 } 216 stage('Backup and Restore Tests') { 217 steps { 218 retry(count: JOB_PROMOTION_RETRIES) { 219 script { 220 build job: "/verrazzano-backup-all-test-oke/${CLEAN_BRANCH_NAME}", 221 parameters: [ 222 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 223 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: 'NONE'), 224 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 225 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 226 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 227 ], wait: true 228 } 229 } 230 } 231 } 232 stage('Kind Acceptance Tests With HA and Grafana DB on 1.27') { 233 steps { 234 retry(count: JOB_PROMOTION_RETRIES) { 235 script { 236 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 237 parameters: [ 238 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 239 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 240 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 241 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 242 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 243 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 244 booleanParam(name: 'INSTALL_HA', value: true), 245 booleanParam(name: 'USE_DB_FOR_GRAFANA', value: true) 246 ], wait: true 247 } 248 } 249 } 250 } 251 stage('Kind Acceptance Tests on 1.24') { 252 steps { 253 retry(count: JOB_PROMOTION_RETRIES) { 254 script { 255 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 256 parameters: [ 257 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.24'), 258 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 259 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 260 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 261 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 262 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 263 booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: true) 264 ], wait: true 265 } 266 } 267 } 268 } 269 stage('Kind Acceptance Tests on 1.25') { 270 steps { 271 retry(count: JOB_PROMOTION_RETRIES) { 272 script { 273 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 274 parameters: [ 275 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.25'), 276 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 277 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 278 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 279 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 280 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 281 booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: true) 282 ], wait: true 283 } 284 } 285 } 286 } 287 stage('Kind Acceptance Tests on 1.26') { 288 steps { 289 retry(count: JOB_PROMOTION_RETRIES) { 290 script { 291 build job: "/verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 292 parameters: [ 293 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.26'), 294 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 295 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 296 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 297 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 298 booleanParam(name: 'ENABLE_JWT_TESTING', value: true), 299 booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: true) 300 ], wait: true 301 } 302 } 303 } 304 } 305 stage('Verrazzano a-la-carte') { 306 steps { 307 retry(count: JOB_PROMOTION_RETRIES) { 308 script { 309 build job: "/verrazzano-a-la-carte/${CLEAN_BRANCH_NAME}", 310 parameters: [ 311 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 312 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 313 ], wait: true 314 } 315 } 316 } 317 } 318 stage('HA tests') { 319 steps { 320 retry(count: JOB_PROMOTION_RETRIES) { 321 script { 322 build job: "/verrazzano-ha-tests/${CLEAN_BRANCH_NAME}", 323 parameters: [ 324 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 325 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 326 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 327 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 328 ], wait: false 329 } 330 } 331 } 332 } 333 stage('Upgrade Resiliency tests') { 334 steps { 335 script { 336 build job: "/verrazzano-upgrade-resiliency-tests/${CLEAN_BRANCH_NAME}", 337 parameters: [ 338 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 339 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 340 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 341 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 342 ], wait: true 343 } 344 } 345 } 346 stage('Uninstall Resiliency tests') { 347 steps { 348 script { 349 build job: "/verrazzano-uninstall-resiliency-suite/${CLEAN_BRANCH_NAME}", 350 parameters: [ 351 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 352 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 353 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 354 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 355 ], wait: true 356 } 357 } 358 } 359 stage('Kind Acceptance Tests on 1.27 Non-Calico') { 360 steps { 361 retry(count: JOB_PROMOTION_RETRIES) { 362 script { 363 // Do not use Calico 364 build job: "verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 365 parameters: [ 366 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 367 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 368 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 369 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 370 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 371 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 372 ], wait: true 373 } 374 } 375 } 376 post { 377 failure { 378 script { 379 TESTS_FAILED = true 380 } 381 } 382 } 383 } 384 stage('Kind Acceptance Tests on 1.27 sslip.io') { 385 steps { 386 retry(count: JOB_PROMOTION_RETRIES) { 387 script { 388 // test with sslip.io 389 build job: "verrazzano-new-kind-acceptance-tests/${CLEAN_BRANCH_NAME}", 390 parameters: [ 391 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 392 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 393 string(name: 'WILDCARD_DNS_DOMAIN', value: 'sslip.io'), 394 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 395 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 396 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 397 ], wait: true 398 } 399 } 400 } 401 post { 402 failure { 403 script { 404 TESTS_FAILED = true 405 } 406 } 407 } 408 } 409 stage('Dynamic Configuration Tests') { 410 steps { 411 retry(count: JOB_PROMOTION_RETRIES) { 412 script { 413 build job: "verrazzano-dynamic-config-suite/${CLEAN_BRANCH_NAME}", 414 parameters: [ 415 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 416 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 417 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 418 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 419 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 420 ], wait: true 421 } 422 } 423 } 424 post { 425 failure { 426 script { 427 TESTS_FAILED = true 428 } 429 } 430 } 431 } 432 stage('OCI DNS/ACME-Staging Tests') { 433 steps { 434 retry(count: JOB_PROMOTION_RETRIES) { 435 script { 436 build job: "verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 437 parameters: [ 438 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 439 string(name: 'CERT_ISSUER', value: "acme"), 440 string(name: 'ACME_ENVIRONMENT', value: "staging"), 441 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 442 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 443 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 444 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 445 ], wait: true 446 } 447 } 448 } 449 post { 450 failure { 451 script { 452 TESTS_FAILED = true 453 } 454 } 455 } 456 } 457 stage('OCI Service Integration Tests') { 458 steps { 459 retry(count: JOB_PROMOTION_RETRIES) { 460 script { 461 build job: "/verrazzano-oci-integration-tests/${CLEAN_BRANCH_NAME}", 462 parameters: [ 463 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 464 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 465 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 466 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 467 ], wait: true 468 } 469 } 470 } 471 post { 472 failure { 473 script { 474 TESTS_FAILED = true 475 } 476 } 477 } 478 } 479 stage('Verrazzano Examples') { 480 steps { 481 retry(count: JOB_PROMOTION_RETRIES) { 482 script { 483 def builtExamples = build job: "/verrazzano-examples/${CLEAN_BRANCH_NAME}", 484 parameters: [ 485 string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'), 486 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 487 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 488 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 489 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 490 ], wait: true 491 // The verrazzano-examples job is the one from which we get the verrazzano images file 492 verrazzanoImagesBuildNumber = builtExamples.number 493 } 494 } 495 } 496 post { 497 success { 498 script { 499 if (verrazzanoImagesBuildNumber > 0) { 500 copyArtifacts(projectName: "${verrazzanoImagesJobProjectName}/${CLEAN_BRANCH_NAME}", 501 selector: specific("${verrazzanoImagesBuildNumber}"), 502 filter: verrazzanoImagesFile) 503 sh """ 504 # Save the images.txt intp the commit specific location, if we have a clean periodic it will copy the images from here to the branch as the current ones 505 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VERSIONED_IMAGES_FILENAME} --file ${verrazzanoImagesFile} 506 """ 507 } else { 508 println("Cannot copy ${verrazzanoImagesFile} from the project ${verrazzanoImagesJobProjectName} - no build number is available to copy from.") 509 } 510 } 511 } 512 failure { 513 script { 514 TESTS_FAILED = true 515 } 516 } 517 } 518 } 519 stage('Verrazzano Distribution and Private Registry') { 520 steps { 521 script { 522 build job: "/verrazzano-distributions-testing/${CLEAN_BRANCH_NAME}", 523 parameters: [ 524 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 525 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 526 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 527 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 528 string(name: 'RUN_TESTS', value: env.IS_PERIODIC_PIPELINE), 529 ], wait: true 530 } 531 } 532 } 533 stage('ClusterAPI cluster creation tests using KIND as admin') { 534 steps { 535 retry(count: JOB_PROMOTION_RETRIES) { 536 script { 537 build job: "verrazzano-clusterAPI-kind-tests/${CLEAN_BRANCH_NAME}", 538 parameters: [ 539 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 540 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 541 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 542 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 543 ], wait: false 544 } 545 } 546 } 547 post { 548 failure { 549 script { 550 TESTS_FAILED = true 551 } 552 } 553 } 554 } 555 stage('OCNE Cluster Driver') { 556 steps { 557 retry(count: JOB_PROMOTION_RETRIES) { 558 script { 559 build job: "verrazzano-ocne-cluster-driver-tests/${CLEAN_BRANCH_NAME}", 560 parameters: [ 561 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 562 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 563 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 564 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 565 ], wait: true 566 } 567 } 568 } 569 post { 570 failure { 571 script { 572 TESTS_FAILED = true 573 } 574 } 575 } 576 } 577 stage('OKE CAPI Cluster Driver') { 578 steps { 579 retry(count: JOB_PROMOTION_RETRIES) { 580 script { 581 build job: "verrazzano-oke-capi-cluster-driver-tests/${CLEAN_BRANCH_NAME}", 582 parameters: [ 583 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 584 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 585 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 586 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS) 587 ], wait: true 588 } 589 } 590 } 591 post { 592 failure { 593 script { 594 TESTS_FAILED = true 595 } 596 } 597 } 598 } 599 } 600 } 601 stage('Release Candidate Validation Checks') { 602 when { 603 allOf { 604 expression { return runTests() } 605 } 606 } 607 steps { 608 script { 609 releaseValidationChecks() 610 } 611 } 612 } 613 stage('Update Last Clean Periodic Test') { 614 environment { 615 GIT_COMMIT_USED = "${env.GIT_COMMIT}" 616 } 617 when { 618 allOf { 619 expression { return runTests() } 620 } 621 } 622 steps { 623 script { 624 sh """ 625 ci/scripts/update_last_clean_periodic_test.sh ${VERRAZZANO_DEV_VERSION} ${SHORT_COMMIT_HASH} 626 """ 627 } 628 } 629 post { 630 success { 631 script { 632 captureDistributionURLs() 633 } 634 } 635 } 636 } 637 } 638 post { 639 always { 640 script { 641 // We don't update any object storage artifacts if this run didn't re-run anything (ie: if it was "UpToDate") 642 if (!periodicsUpToDate && !periodicsUpToDateFailed) { 643 sh """ 644 # Update the last periodic commit 645 echo "git-commit=${env.GIT_COMMIT}" > commit-used.txt 646 cat commit-used.txt 647 oci --region ${OCI_OS_REGION} os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${LAST_PERIODIC_OS_LOCATION} --file commit-used.txt 648 """ 649 } else { 650 println("Skipping update of the last periodic commit used. This run was marked UP-TO-DATE so it is already there") 651 } 652 } 653 archiveArtifacts artifacts: "**/prerelease_validation.out,**/release_status.out,**/${verrazzanoImagesFile},**/${verrazzanoDistributionsFile}", allowEmptyArchive: true 654 } 655 failure { 656 script { 657 failedOrAborted() 658 } 659 } 660 aborted { 661 script { 662 failedOrAborted() 663 } 664 } 665 cleanup { 666 deleteDir() 667 } 668 } 669 } 670 671 def failedOrAborted() { 672 if (isAlertingEnabled()) { 673 if (isPagerDutyEnabled()) { 674 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", 675 incDescription: "Verrazzano Periodic Tests: ${env.JOB_NAME} - Failed", 676 incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 677 } 678 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST} ${PIPELINE_OWNERS}\n\nChange comparison: ${COMPARISON_URL_ON_FAILURE}" ) 679 echo "done alerts" 680 } 681 } 682 683 // Preliminary job checks and display updates 684 def preliminaryChecks() { 685 // Get the last stable commit ID to pass the triggered tests 686 def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}" 687 GIT_COMMIT_TO_USE = stableCommitProps['git-commit'] 688 echo "Last stable commit: ${GIT_COMMIT_TO_USE}" 689 690 LAST_CLEAN_PERIODIC_COMMIT=getLastCleanPeriodicCommit() 691 echo "Last clean periodics commit: ${LAST_CLEAN_PERIODIC_COMMIT}" 692 693 if (LAST_CLEAN_PERIODIC_COMMIT == GIT_COMMIT_TO_USE) { 694 // If we had a clean periodic run and the commit hasn't changed 695 periodicsUpToDate = true 696 } else { 697 // Check if we are still at the same commit previously run (if so we know it wasn't clean and it failed in some way) 698 LAST_PERIODIC_RUN_COMMIT=getLastPeriodicRunCommit() 699 if (LAST_PERIODIC_RUN_COMMIT != null && LAST_PERIODIC_RUN_COMMIT == GIT_COMMIT_TO_USE) { 700 periodicsUpToDateFailed = true 701 } 702 } 703 704 echo "Up to date: ${periodicsUpToDate}" 705 echo "Dry run: ${params.DRY_RUN}" 706 echo "Force run: ${params.FORCE}" 707 echo "Execute tests: " + runTests() 708 709 // Indicate in title if run is up-to-date or dry-run 710 if (params.DRY_RUN) { 711 currentBuild.displayName = "${currentBuild.displayName} : DRY-RUN" 712 } 713 if (periodicsUpToDate) { 714 currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE" 715 } 716 717 if (params.FORCE) { 718 currentBuild.displayName = "${currentBuild.displayName} : FORCE" 719 } else if (periodicsUpToDateFailed) { 720 currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE-FAILED" 721 currentBuild.result = 'FAILURE' 722 error('Failing the build since the current commit matches the commit of previously failing periodic build') 723 } 724 725 if (runTests()) { 726 echo "Executing periodic tests for commit ${GIT_COMMIT_TO_USE}" 727 } 728 } 729 730 def dockerLogins() { 731 try { 732 sh """ 733 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 734 """ 735 } catch(error) { 736 echo "docker login failed, retrying after sleep" 737 retry(4) { 738 sleep(30) 739 sh """ 740 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 741 """ 742 } 743 } 744 if (!(env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) { 745 try { 746 sh """ 747 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 748 """ 749 } catch(error) { 750 echo "docker login failed, retrying after sleep" 751 retry(4) { 752 sleep(30) 753 sh """ 754 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 755 """ 756 } 757 } 758 } 759 } 760 761 def scmCheckout() { 762 echo "${NODE_LABELS}" 763 echo "SCM checkout of ${GIT_COMMIT_TO_USE}" 764 def scmInfo = checkout([ 765 $class: 'GitSCM', 766 branches: [[name: GIT_COMMIT_TO_USE]], 767 doGenerateSubmoduleConfigurations: false, 768 extensions: [], 769 submoduleCfg: [], 770 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 771 env.GIT_COMMIT = scmInfo.GIT_COMMIT 772 env.GIT_BRANCH = scmInfo.GIT_BRANCH 773 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 774 // If the commit we were handed is not what the SCM says we are using, fail 775 if (!env.GIT_COMMIT.equals(GIT_COMMIT_TO_USE)) { 776 error( "SCM didn't checkout the commit we expected. Expected: ${GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}") 777 } 778 779 if (LAST_CLEAN_PERIODIC_COMMIT != null) { 780 COMPARISON_URL_ON_FAILURE = "https://github.com/verrazzano/verrazzano/compare/${LAST_CLEAN_PERIODIC_COMMIT}...${GIT_COMMIT_TO_USE}" 781 def lastClean = "${LAST_CLEAN_PERIODIC_COMMIT}" 782 def currentStable = "${GIT_COMMIT_TO_USE}" 783 def commitList = getCommitListFromGitLog(lastClean, currentStable) 784 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 785 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 786 SUSPECT_LIST = getSuspectList(commitList, userMappings) 787 echo "Suspect list: ${SUSPECT_LIST}" 788 } 789 } 790 echo "URL if fails: ${COMPARISON_URL_ON_FAILURE}" 791 } 792 793 def cleanWorkspaceAndCheckout() { 794 scmCheckout() 795 dockerLogins() 796 def props = readProperties file: '.verrazzano-development-version' 797 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 798 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 799 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 800 // update the description with some meaningful info 801 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + GIT_COMMIT_TO_USE 802 storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 803 fullBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 804 liteBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 805 VERSIONED_IMAGES_FILENAME = "verrazzano_${VERRAZZANO_DEV_VERSION}-images.txt" 806 } 807 808 // Returns the last clean commit for the periodics, or null if the commit file does not exist yet. 809 // - fails the pipeline if any error other than 404 is returned by the OCI CLI 810 def getLastCleanPeriodicCommit() { 811 lastPeriodicCommitCommandOutput = sh ( 812 label: "Get last clean periodic commit ID", 813 script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_PERIODIC_OS_LOCATION} --file ${CLEAN_PERIODIC_LOCATION} 2>&1 || true", 814 returnStdout: true 815 ).trim() 816 echo "command out: ${lastPeriodicCommitCommandOutput}" 817 if (lastPeriodicCommitCommandOutput.length() > 0) { 818 // We can get warning messages here as well even when the command succeeded, so be more precise on the checking 819 if (lastPeriodicCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) { 820 // If we think we had a status: NNN, we ignore 404 and fail for others 821 assert lastPeriodicCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 822 } else { 823 // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error 824 sh """ 825 if [ ! -f ${CLEAN_PERIODIC_LOCATION} ]; then 826 echo "An unexpected error occurred getting last clean periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 827 exit 1 828 fi 829 """ 830 } 831 } 832 // Get the commit ID for the last known clean pass of the Periodic tests 833 def cleanPeriodicsCommitProps = readProperties file: "${CLEAN_PERIODIC_LOCATION}" 834 return cleanPeriodicsCommitProps['git-commit'] 835 } 836 837 // Returns the last run commit for the periodics, or null if the commit file does not exist yet. 838 // - fails the pipeline if any error other than 404 is returned by the OCI CLI 839 def getLastPeriodicRunCommit() { 840 lastPeriodicCommitCommandOutput = sh ( 841 label: "Get last clean periodic commit ID", 842 script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${LAST_PERIODIC_OS_LOCATION} --file ${LAST_PERIODIC_LOCATION} 2>&1 || true", 843 returnStdout: true 844 ).trim() 845 echo "command out: ${lastPeriodicCommitCommandOutput}" 846 if (lastPeriodicCommitCommandOutput.length() > 0) { 847 // We can get warning messages here as well even when the command succeeded, so be more precise on the checking 848 if (lastPeriodicCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) { 849 // If we think we had a status: NNN, we ignore 404 and fail for others 850 assert lastPeriodicCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 851 } else { 852 // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error 853 sh """ 854 if [ ! -f ${LAST_PERIODIC_LOCATION} ]; then 855 echo "An unexpected error occurred getting last periodic run commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 856 exit 1 857 fi 858 """ 859 } 860 } 861 // Get the commit ID for the last known clean pass of the Periodic tests 862 def lastPeriodicsCommitProps = readProperties file: "${LAST_PERIODIC_LOCATION}" 863 return lastPeriodicsCommitProps['git-commit'] 864 } 865 866 // Checks all the conditions gating test execution and collates the result 867 def runTests() { 868 return params.FORCE || ( ! periodicsUpToDate && ! periodicsUpToDateFailed && ! params.DRY_RUN ) 869 } 870 871 def isAlertingEnabled() { 872 // this controls whether any alerting happens for these tests 873 if (NOTIFY_PERIODIC_FAILURES.equals("true") && (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) { 874 echo "Alert notifications enabled for ${env.BRANCH_NAME}" 875 return true 876 } 877 return false 878 } 879 880 def releaseValidationChecks() { 881 def built = build job: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 882 parameters: [ 883 string(name: 'COMMIT_TO_USE', value: env.GIT_COMMIT) 884 ], wait: true, propagate: false 885 println("Result of verrazzano-prerelease-check is ${built.result}") 886 dir ("${WORKSPACE}") { 887 copyArtifacts(projectName: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 888 selector: specific("${built.number}")); 889 def releaseStatus = readFile file: "release_status.out" 890 currentBuild.displayName = "${currentBuild.displayName} : ${releaseStatus}" 891 } 892 } 893 894 def isPagerDutyEnabled() { 895 // this additionally controls whether PD alerts are enabled (note that you must also enable alerting in general as well if you want these) 896 if (NOTIFY_PAGERDUTY_PERIODIC_FAILURES.equals("true")) { 897 echo "Pager-Duty notifications enabled via global override setting" 898 return true 899 } 900 return false 901 } 902 903 // def getCronSchedule() { 904 // if (env.BRANCH_NAME.equals("master")) { 905 // return "H */6 * * *" 906 // } else if (env.BRANCH_NAME.startsWith("release-")) { 907 // return "@daily" 908 // } 909 // return "" 910 // } 911 912 // Called in Stage Clean workspace and checkout steps 913 def getCommitListFromGitLog(lastClean, currentStable) { 914 echo "Checking for change sets" 915 def commitList = sh(returnStdout: true, script: "git log ${lastClean}...${currentStable} --oneline | cut -d \" \" -f 1").trim().split('\n') 916 for (int i = 0; i < commitList.size(); i++) { 917 echo "Found commit id: ${commitList[i]}" 918 } 919 return commitList 920 } 921 922 def trimIfGithubNoreplyUser(userIn) { 923 if (userIn == null) { 924 echo "Not a github noreply user, not trimming: ${userIn}" 925 return userIn 926 } 927 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 928 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 929 return userOut; 930 } 931 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 932 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 933 return userOut; 934 } 935 if (userIn.matches(".*@users.noreply.github.com")) { 936 def userOut = userIn.substring(0, userIn.indexOf("@")) 937 return userOut; 938 } 939 echo "Not a github noreply user, not trimming: ${userIn}" 940 return userIn 941 } 942 943 def getSuspectList(commitList, userMappings) { 944 def retValue = "" 945 def suspectList = [] 946 if (commitList == null || commitList.size() == 0) { 947 echo "No commits to form suspect list" 948 } else { 949 for (int i = 0; i < commitList.size(); i++) { 950 def id = commitList[i] 951 try { 952 def gitAuthor = sh( 953 script: "git log --format='%ae' '$id^!'", 954 returnStdout: true 955 ).trim() 956 if (gitAuthor != null) { 957 def author = trimIfGithubNoreplyUser(gitAuthor) 958 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 959 if (userMappings.containsKey(author)) { 960 def slackUser = userMappings.get(author) 961 if (!suspectList.contains(slackUser)) { 962 echo "Added ${slackUser} as suspect" 963 retValue += " ${slackUser}" 964 suspectList.add(slackUser) 965 } 966 } else { 967 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 968 if (!suspectList.contains(author)) { 969 echo "Added ${author} as suspect" 970 retValue += " ${author}" 971 suspectList.add(author) 972 } 973 } 974 } else { 975 echo "No author returned from git" 976 } 977 } catch (Exception e) { 978 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 979 } 980 } 981 } 982 def startedByUser = ""; 983 def causes = currentBuild.getBuildCauses() 984 echo "causes: " + causes.toString() 985 for (cause in causes) { 986 def causeString = cause.toString() 987 echo "current cause: " + causeString 988 def causeInfo = readJSON text: causeString 989 if (causeInfo.userId != null) { 990 startedByUser = causeInfo.userId 991 } 992 } 993 994 if (startedByUser.length() > 0) { 995 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 996 def author = trimIfGithubNoreplyUser(startedByUser) 997 echo "DEBUG: author: ${startedByUser}, ${author}" 998 if (userMappings.containsKey(author)) { 999 def slackUser = userMappings.get(author) 1000 if (!suspectList.contains(slackUser)) { 1001 echo "Added ${slackUser} as suspect" 1002 retValue += " ${slackUser}" 1003 suspectList.add(slackUser) 1004 } 1005 } else { 1006 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 1007 if (!suspectList.contains(author)) { 1008 echo "Added ${author} as suspect" 1009 retValue += " ${author}" 1010 suspectList.add(author) 1011 } 1012 } 1013 } else { 1014 echo "Build not started by a user, not adding to notification list" 1015 } 1016 echo "returning suspect list: ${retValue}" 1017 return retValue 1018 } 1019 1020 @NonCPS 1021 List extractReleaseTags(final String fileContent) { 1022 List releases = [] 1023 fileContent.eachLine { tag -> 1024 releases << tag 1025 } 1026 return releases 1027 } 1028 1029 def getLatestReleaseVersion() { 1030 final String releaseTags = readFile(file: "${workspace}/tags.txt") 1031 list gitTags = extractReleaseTags(releaseTags) 1032 echo "gitTags = ${gitTags}" 1033 return gitTags.pop() 1034 } 1035 1036 // Create a html file containing the links to the Verrazzano distributions 1037 def captureDistributionURLs() { 1038 script { 1039 def BRANCH_IN_OS = "${env.BRANCH_NAME.replace("/", "%252F")}" 1040 BASE_URL="https://objectstorage.${OCI_OS_REGION}.oraclecloud.com/n/${OCI_OS_NAMESPACE}/b/${OCI_OS_BUCKET}/o/${BRANCH_IN_OS}-last-clean-periodic-test" 1041 LITE_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 1042 FULL_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 1043 sh """ 1044 cat <<EOF > ${WORKSPACE}/${verrazzanoDistributionsFile} 1045 <!DOCTYPE html> 1046 <html> 1047 <body> 1048 <b>Verrazzano Release Distributions</b> 1049 <ul> 1050 <li><a href="${LITE_BUNDLE}">Verrazzano Lite Distribution</a></li> 1051 <li><a href="${FULL_BUNDLE}">Verrazzano Full Distribution</a></li> 1052 </ul> 1053 </body> 1054 </html> 1055 EOF 1056 """ 1057 } 1058 }