github.com/verrazzano/verrazzano@v1.7.0/ci/oke-ocidns/JenkinsfileDnsSweepTests (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 import groovy.transform.Field 5 6 @Field 7 def GIT_COMMIT_TO_USE = "" 8 @Field 9 def LAST_CLEAN_PERIODIC_COMMIT = "" 10 @Field 11 def VERRAZZANO_DEV_VERSION = "" 12 @Field 13 def RELEASABLE_IMAGES_OBJECT_STORE = "" 14 @Field 15 def TESTS_FAILED = false 16 @Field 17 def storeLocation="" 18 @Field 19 def verrazzanoPrefix="verrazzano-" 20 @Field 21 def fullBundle="" 22 @Field 23 def liteBundle="" 24 @Field 25 def SUSPECT_LIST = "" 26 @Field 27 def COMPARISON_URL_ON_FAILURE = "" 28 29 // The job name from which the verrazzano_images file is available to be copied to this job 30 // We will copy over and make it part of the artifacts of the periodic job, available when we want to release a candidate 31 @Field 32 def verrazzanoImagesJobProjectName = "verrazzano-examples" 33 @Field 34 def verrazzanoImagesFile = "verrazzano_images.txt" 35 @Field 36 def verrazzanoImagesBuildNumber = 0 // will be set to actual build number when the job is run 37 @Field 38 def periodicsUpToDate = false // If true, indicates that the periodics already passed at the latest commit 39 40 // Non Fields 41 def branchSpecificSchedule = getCronSchedule() 42 43 // File containing the links to download the Verrazzano distributions 44 @Field 45 def verrazzanoDistributionsFile = "verrazzano_distributions.html" 46 47 pipeline { 48 options { 49 skipDefaultCheckout true 50 disableConcurrentBuilds() 51 timestamps () 52 } 53 54 agent { 55 docker { 56 image "${RUNNER_DOCKER_IMAGE}" 57 args "${RUNNER_DOCKER_ARGS}" 58 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 59 registryCredentialsId 'ocir-pull-and-push-account' 60 label "pipeline-job-large" 61 } 62 } 63 64 65 parameters { 66 string (name: 'GIT_COMMIT_TO_USE', 67 defaultValue: 'NONE', 68 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 69 trim: true) 70 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 71 defaultValue: 'NONE', 72 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 73 trim: true) 74 string (name: 'TAGGED_TESTS', 75 defaultValue: '', 76 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 77 trim: true) 78 string (name: 'INCLUDED_TESTS', 79 defaultValue: '.*', 80 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 81 trim: true) 82 string (name: 'EXCLUDED_TESTS', 83 defaultValue: '_excluded_test', 84 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 85 trim: true) 86 booleanParam (description: 'Force execution of the tests even if up-to-date', name: 'FORCE', defaultValue: false) 87 booleanParam (description: 'Skip test execution (for debugging)', name: 'DRY_RUN', defaultValue: false) 88 booleanParam (description: 'Run KinD Based DNS tests ', name: 'KIND_DNS_TESTS', defaultValue: true) 89 booleanParam (description: 'Run OKE Based DNS tests ', name: 'OKE_DNS_TESTS', defaultValue: false) 90 } 91 92 environment { 93 OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment') 94 OCIR_SCAN_TARGET = credentials('ocir-scan-target') 95 OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry') 96 OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path') 97 DOCKER_SCAN_CREDS = credentials('v8odev-ocir') 98 DOCKER_CREDS = credentials('github-packages-credentials-rw') 99 DOCKER_REPO = 'ghcr.io' 100 101 OCI_CLI_AUTH="instance_principal" 102 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 103 OCI_OS_BUCKET="verrazzano-builds" 104 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 105 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 106 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 107 108 STABLE_COMMIT_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-stable-commit.txt" 109 CLEAN_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test/verrazzano_periodic-commit.txt" 110 111 STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt" 112 CLEAN_PERIODIC_LOCATION = "${WORKSPACE}/last-clean-periodic-commit.txt" 113 114 OCI_OS_REGION="us-phoenix-1" 115 } 116 117 // This job runs against the latest stable master commit. That is defined as the last clean master build and test run whose 118 // commit has been stored in object storage. This job will fetch that commit from master and run extended tests using that. 119 // This job is NOT currently setup to run extended tests from other branches, if you need to run those extended jobs you will 120 // need to run those against your branch individually. 121 122 stages { 123 stage('Clean workspace and checkout') { 124 steps { 125 sh """ 126 echo "${NODE_LABELS}" 127 """ 128 script { 129 if (params.GIT_COMMIT_TO_USE == "NONE") { 130 echo "Specific GIT commit was not specified, use current head" 131 def scmInfo = checkout([ 132 $class: 'GitSCM', 133 branches: [[name: env.BRANCH_NAME]], 134 doGenerateSubmoduleConfigurations: false, 135 extensions: [], 136 submoduleCfg: [], 137 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 138 env.GIT_COMMIT = scmInfo.GIT_COMMIT 139 env.GIT_BRANCH = scmInfo.GIT_BRANCH 140 } else { 141 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 142 def scmInfo = checkout([ 143 $class: 'GitSCM', 144 branches: [[name: params.GIT_COMMIT_TO_USE]], 145 doGenerateSubmoduleConfigurations: false, 146 extensions: [], 147 submoduleCfg: [], 148 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 149 env.GIT_COMMIT = scmInfo.GIT_COMMIT 150 env.GIT_BRANCH = scmInfo.GIT_BRANCH 151 // If the commit we were handed is not what the SCM says we are using, fail 152 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 153 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 154 exit 1 155 } 156 } 157 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 158 } 159 160 script { 161 def props = readProperties file: '.verrazzano-development-version' 162 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 163 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 164 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 165 // update the description with some meaningful info 166 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 167 def currentCommitHash = env.GIT_COMMIT 168 def commitList = getCommitList() 169 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 170 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 171 SUSPECT_LIST = getSuspectList(commitList, userMappings) 172 echo "Suspect list: ${SUSPECT_LIST}" 173 } 174 } 175 } 176 } 177 178 stage ('All DNS Tests with KinD') { 179 when { 180 allOf { 181 expression { return runKindDnsTests() } 182 } 183 } 184 parallel { 185 stage('oci user principal with global dns') { 186 steps { 187 retry(count: JOB_PROMOTION_RETRIES) { 188 script { 189 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 190 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 191 parameters: [ 192 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 193 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 194 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 195 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 196 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 197 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 198 ], wait: true 199 } 200 } 201 } 202 } 203 204 stage('oci instance principal with global dns') { 205 steps { 206 retry(count: JOB_PROMOTION_RETRIES) { 207 script { 208 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 209 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 210 parameters: [ 211 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 212 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 213 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 214 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 215 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 216 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 217 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 218 ], wait: true 219 } 220 } 221 } 222 } 223 224 stage('oci user principal with private dns') { 225 steps { 226 retry(count: JOB_PROMOTION_RETRIES) { 227 script { 228 229 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 230 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 231 parameters: [ 232 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 233 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 234 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 235 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 236 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 237 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 238 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 239 ], wait: true 240 } 241 } 242 } 243 } 244 245 stage('oci instance principal with private dns') { 246 steps { 247 retry(count: JOB_PROMOTION_RETRIES) { 248 script { 249 250 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 251 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 252 parameters: [ 253 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 254 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 255 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 256 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 257 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 258 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 259 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 260 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 261 ], wait: true 262 } 263 } 264 } 265 } 266 267 stage('oci user principal with global dns cert-issuer acme and env as staging') { 268 steps { 269 retry(count: JOB_PROMOTION_RETRIES) { 270 script { 271 build job: "verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 272 parameters: [ 273 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 274 string(name: 'CERT_ISSUER', value: "acme"), 275 string(name: 'ACME_ENVIRONMENT', value: "staging"), 276 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 277 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 278 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 279 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 280 ], wait: true 281 } 282 } 283 } 284 post { 285 failure { 286 script { 287 TESTS_FAILED = true 288 } 289 } 290 } 291 } 292 293 294 } 295 } 296 297 298 299 300 stage ('All DNS Tests with OKE') { 301 when { 302 allOf { 303 expression { return runOkeDnsTests() } 304 } 305 } 306 parallel { 307 308 stage('oci user principal with global dns and private istio lb') { 309 steps { 310 retry(count: JOB_PROMOTION_RETRIES) { 311 script { 312 313 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 314 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 315 parameters: [ 316 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 317 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 318 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 319 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 320 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 321 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 322 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 323 ], wait: true 324 } 325 } 326 } 327 } 328 329 stage('oci user principal with global dns and private nginx lb') { 330 steps { 331 retry(count: JOB_PROMOTION_RETRIES) { 332 script { 333 334 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 335 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 336 parameters: [ 337 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 338 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 339 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 340 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 341 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 342 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 343 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 344 ], wait: true 345 } 346 } 347 } 348 } 349 350 stage('oci user principal with global dns and private istio and nginx lb') { 351 steps { 352 retry(count: JOB_PROMOTION_RETRIES) { 353 script { 354 355 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 356 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 357 parameters: [ 358 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 359 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 360 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 361 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 362 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 363 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 364 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 365 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 366 ], wait: true 367 } 368 } 369 } 370 } 371 372 373 374 stage('oci instance principal with global dns and private istio lb') { 375 steps { 376 retry(count: JOB_PROMOTION_RETRIES) { 377 script { 378 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 379 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 380 parameters: [ 381 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 382 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 383 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 384 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 385 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 386 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 387 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 388 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 389 ], wait: true 390 } 391 } 392 } 393 } 394 395 stage('oci instance principal with global dns and private nginx lb') { 396 steps { 397 retry(count: JOB_PROMOTION_RETRIES) { 398 script { 399 400 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 401 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 402 parameters: [ 403 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 404 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 405 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 406 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 407 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 408 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 409 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 410 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 411 ], wait: true 412 } 413 } 414 } 415 } 416 417 stage('oci instance principal with global dns and private istio and nginx lb') { 418 steps { 419 retry(count: JOB_PROMOTION_RETRIES) { 420 script { 421 422 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 423 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 424 parameters: [ 425 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 426 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 427 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 428 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 429 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 430 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 431 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 432 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 433 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 434 ], wait: true 435 } 436 } 437 } 438 } 439 440 441 442 443 444 445 stage('oci user principal with private dns and private istio lb') { 446 steps { 447 retry(count: JOB_PROMOTION_RETRIES) { 448 script { 449 450 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 451 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 452 parameters: [ 453 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 454 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 455 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 456 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 457 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 458 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 459 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 460 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 461 ], wait: true 462 } 463 } 464 } 465 } 466 467 stage('oci user principal with private dns and private nginx lb') { 468 steps { 469 retry(count: JOB_PROMOTION_RETRIES) { 470 script { 471 472 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 473 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 474 parameters: [ 475 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 476 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 477 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 478 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 479 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 480 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 481 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 482 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 483 ], wait: true 484 } 485 } 486 } 487 } 488 489 stage('oci user principal with private dns and private istio and nginx lb') { 490 steps { 491 retry(count: JOB_PROMOTION_RETRIES) { 492 script { 493 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 494 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 495 parameters: [ 496 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 497 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 498 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 499 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 500 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 501 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 502 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 503 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 504 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 505 ], wait: true 506 } 507 } 508 } 509 } 510 511 512 513 stage('oci instance principal with private dns and private istio lb') { 514 steps { 515 retry(count: JOB_PROMOTION_RETRIES) { 516 script { 517 518 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 519 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 520 parameters: [ 521 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 522 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 523 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 524 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 525 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 526 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 527 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 528 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 529 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 530 ], wait: true 531 } 532 } 533 } 534 } 535 536 stage('oci instance principal with private dns and private nginx lb') { 537 steps { 538 retry(count: JOB_PROMOTION_RETRIES) { 539 script { 540 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 541 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 542 parameters: [ 543 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 544 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 545 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 546 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 547 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 548 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 549 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 550 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 551 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 552 ], wait: true 553 } 554 } 555 } 556 } 557 558 stage('oci instance principal with private dns and private istio and nginx lb') { 559 steps { 560 retry(count: JOB_PROMOTION_RETRIES) { 561 script { 562 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 563 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 564 parameters: [ 565 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 566 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 567 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 568 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 569 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 570 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 571 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 572 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 573 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 574 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 575 ], wait: true 576 } 577 } 578 } 579 } 580 } 581 } 582 } 583 post { 584 always { 585 script { 586 if (verrazzanoImagesBuildNumber > 0) { 587 copyArtifacts(projectName: "${verrazzanoImagesJobProjectName}/${CLEAN_BRANCH_NAME}", 588 selector: specific("${verrazzanoImagesBuildNumber}"), 589 filter: verrazzanoImagesFile) 590 sh """ 591 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BRANCH_NAME}/${RELEASABLE_IMAGES_OBJECT_STORE} --file ${verrazzanoImagesFile} 592 """ 593 } else { 594 println("Cannot copy ${verrazzanoImagesFile} from the project ${verrazzanoImagesJobProjectName} - no build number is available to copy from.") 595 } 596 } 597 archiveArtifacts artifacts: "**/prerelease_validation.out,**/release_status.out,**/${verrazzanoImagesFile},**/${verrazzanoDistributionsFile}", allowEmptyArchive: true 598 } 599 failure { 600 script { 601 if (isAlertingEnabled()) { 602 if (isPagerDutyEnabled()) { 603 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", 604 incDescription: "Verrazzano Periodic Tests: ${env.JOB_NAME} - Failed", 605 incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 606 } 607 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}\n\nChange comparison: ${COMPARISON_URL_ON_FAILURE}" ) 608 echo "done alerts" 609 } 610 } 611 } 612 cleanup { 613 deleteDir() 614 } 615 } 616 } 617 618 // Preliminary job checks and display updates 619 def preliminaryChecks() { 620 // Get the last stable commit ID to pass the triggered tests 621 def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}" 622 GIT_COMMIT_TO_USE = stableCommitProps['git-commit'] 623 echo "Last stable commit: ${GIT_COMMIT_TO_USE}" 624 625 LAST_CLEAN_PERIODIC_COMMIT=getLastCleanPeriodicCommit() 626 echo "Last clean periodics commit: ${LAST_CLEAN_PERIODIC_COMMIT}" 627 628 if (LAST_CLEAN_PERIODIC_COMMIT == GIT_COMMIT_TO_USE) { 629 periodicsUpToDate = true 630 } 631 632 echo "Up to date: ${periodicsUpToDate}" 633 echo "Dry run: ${params.DRY_RUN}" 634 echo "Force run: ${params.FORCE}" 635 echo "Execute tests: " + runTests() 636 637 // Indicate in title if run is up-to-date or dry-run 638 if (params.DRY_RUN) { 639 currentBuild.displayName = "${currentBuild.displayName} : DRY-RUN" 640 } 641 if (periodicsUpToDate) { 642 currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE" 643 } 644 if (params.FORCE) { 645 currentBuild.displayName = "${currentBuild.displayName} : FORCE" 646 } 647 648 if (runTests()) { 649 echo "Executing periodic tests for commit ${GIT_COMMIT_TO_USE}" 650 } 651 } 652 653 def dockerLogins() { 654 try { 655 sh """ 656 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 657 """ 658 } catch(error) { 659 echo "docker login failed, retrying after sleep" 660 retry(4) { 661 sleep(30) 662 sh """ 663 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 664 """ 665 } 666 } 667 if (!(env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-1."))) { 668 try { 669 sh """ 670 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 671 """ 672 } catch(error) { 673 echo "docker login failed, retrying after sleep" 674 retry(4) { 675 sleep(30) 676 sh """ 677 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 678 """ 679 } 680 } 681 } 682 } 683 684 def scmCheckout() { 685 echo "${NODE_LABELS}" 686 echo "SCM checkout of ${GIT_COMMIT_TO_USE}" 687 def scmInfo = checkout([ 688 $class: 'GitSCM', 689 branches: [[name: GIT_COMMIT_TO_USE]], 690 doGenerateSubmoduleConfigurations: false, 691 extensions: [], 692 submoduleCfg: [], 693 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 694 env.GIT_COMMIT = scmInfo.GIT_COMMIT 695 env.GIT_BRANCH = scmInfo.GIT_BRANCH 696 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 697 // If the commit we were handed is not what the SCM says we are using, fail 698 if (!env.GIT_COMMIT.equals(GIT_COMMIT_TO_USE)) { 699 error( "SCM didn't checkout the commit we expected. Expected: ${GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}") 700 } 701 702 if (LAST_CLEAN_PERIODIC_COMMIT != null) { 703 COMPARISON_URL_ON_FAILURE = "https://github.com/verrazzano/verrazzano/compare/${LAST_CLEAN_PERIODIC_COMMIT}...${GIT_COMMIT_TO_USE}" 704 def lastClean = "${LAST_CLEAN_PERIODIC_COMMIT}" 705 def currentStable = "${GIT_COMMIT_TO_USE}" 706 def commitList = getCommitListFromGitLog(lastClean, currentStable) 707 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 708 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 709 SUSPECT_LIST = getSuspectList(commitList, userMappings) 710 echo "Suspect list: ${SUSPECT_LIST}" 711 } 712 } 713 echo "URL if fails: ${COMPARISON_URL_ON_FAILURE}" 714 } 715 716 def cleanWorkspaceAndCheckout() { 717 scmCheckout() 718 dockerLogins() 719 def props = readProperties file: '.verrazzano-development-version' 720 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 721 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 722 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 723 // update the description with some meaningful info 724 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + GIT_COMMIT_TO_USE 725 storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 726 fullBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 727 liteBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 728 RELEASABLE_IMAGES_OBJECT_STORE = "verrazzano_${VERRAZZANO_DEV_VERSION}-images.txt" 729 } 730 731 // Returns the last clean commit for the periodics, or null if the commit file does not exist yet. 732 // - fails the pipeline if any error other than 404 is returned by the OCI CLI 733 def getLastCleanPeriodicCommit() { 734 lastPeriodicCommitCommandOutput = sh ( 735 label: "Get last clean periodic commit ID", 736 script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_PERIODIC_OS_LOCATION} --file ${CLEAN_PERIODIC_LOCATION} 2>&1 || true", 737 returnStdout: true 738 ).trim() 739 echo "command out: ${lastPeriodicCommitCommandOutput}" 740 if (lastPeriodicCommitCommandOutput.length() > 0) { 741 // We can get warning messages here as well even when the command succeeded, so be more precise on the checking 742 if (lastPeriodicCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) { 743 // If we think we had a status: NNN, we ignore 404 and fail for others 744 assert lastPeriodicCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 745 } else { 746 // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error 747 sh """ 748 if [ ! -f ${CLEAN_PERIODIC_LOCATION} ]; then 749 echo "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 750 exit 1 751 fi 752 """ 753 } 754 } 755 // Get the commit ID for the last known clean pass of the Periodic tests 756 def cleanPeriodicsCommitProps = readProperties file: "${CLEAN_PERIODIC_LOCATION}" 757 return cleanPeriodicsCommitProps['git-commit'] 758 } 759 760 // Checks all the conditions gating test execution and coallates the result 761 def runTests() { 762 return params.FORCE || ( ! periodicsUpToDate && ! params.DRY_RUN ) 763 } 764 765 def runKindDnsTests() { 766 return params.KIND_DNS_TESTS && ( ! params.DRY_RUN ) 767 } 768 769 def runOkeDnsTests() { 770 return params.OKE_DNS_TESTS && ( ! params.DRY_RUN ) 771 } 772 773 def isAlertingEnabled() { 774 // this controls whether any alerting happens for these tests 775 if (NOTIFY_PERIODIC_FAILURES.equals("true") && (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-1."))) { 776 echo "Alert notifications enabled for ${env.BRANCH_NAME}" 777 return true 778 } 779 return false 780 } 781 782 def releaseValidationChecks() { 783 def built = build job: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 784 parameters: [ 785 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT) 786 ], wait: true, propagate: false 787 println("Result of verrazzano-prerelease-check is ${built.result}") 788 dir ("${WORKSPACE}") { 789 copyArtifacts(projectName: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 790 selector: specific("${built.number}")); 791 def releaseStatus = readFile file: "release_status.out" 792 currentBuild.displayName = "${currentBuild.displayName} : ${releaseStatus}" 793 } 794 } 795 796 def isPagerDutyEnabled() { 797 // this additionally controls whether PD alerts are enabled (note that you must also enable alerting in general as well if you want these) 798 if (NOTIFY_PAGERDUTY_PERIODIC_FAILURES.equals("true")) { 799 echo "Pager-Duty notifications enabled via global override setting" 800 return true 801 } 802 return false 803 } 804 805 def getCronSchedule() { 806 if (env.BRANCH_NAME.equals("master")) { 807 return "H */6 * * *" 808 } else if (env.BRANCH_NAME.startsWith("release-1")) { 809 return "@daily" 810 } 811 return "" 812 } 813 814 // Called in Stage Clean workspace and checkout steps 815 def getCommitListFromGitLog(lastClean, currentStable) { 816 echo "Checking for change sets" 817 def commitList = sh(returnStdout: true, script: "git log ${lastClean}...${currentStable} --oneline | cut -d \" \" -f 1").trim().split('\n') 818 for (int i = 0; i < commitList.size(); i++) { 819 echo "Found commit id: ${commitList[i]}" 820 } 821 return commitList 822 } 823 824 def trimIfGithubNoreplyUser(userIn) { 825 if (userIn == null) { 826 echo "Not a github noreply user, not trimming: ${userIn}" 827 return userIn 828 } 829 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 830 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 831 return userOut; 832 } 833 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 834 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 835 return userOut; 836 } 837 if (userIn.matches(".*@users.noreply.github.com")) { 838 def userOut = userIn.substring(0, userIn.indexOf("@")) 839 return userOut; 840 } 841 echo "Not a github noreply user, not trimming: ${userIn}" 842 return userIn 843 } 844 845 def getSuspectList(commitList, userMappings) { 846 def retValue = "" 847 def suspectList = [] 848 if (commitList == null || commitList.size() == 0) { 849 echo "No commits to form suspect list" 850 } else { 851 for (int i = 0; i < commitList.size(); i++) { 852 def id = commitList[i] 853 try { 854 def gitAuthor = sh( 855 script: "git log --format='%ae' '$id^!'", 856 returnStdout: true 857 ).trim() 858 if (gitAuthor != null) { 859 def author = trimIfGithubNoreplyUser(gitAuthor) 860 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 861 if (userMappings.containsKey(author)) { 862 def slackUser = userMappings.get(author) 863 if (!suspectList.contains(slackUser)) { 864 echo "Added ${slackUser} as suspect" 865 retValue += " ${slackUser}" 866 suspectList.add(slackUser) 867 } 868 } else { 869 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 870 if (!suspectList.contains(author)) { 871 echo "Added ${author} as suspect" 872 retValue += " ${author}" 873 suspectList.add(author) 874 } 875 } 876 } else { 877 echo "No author returned from git" 878 } 879 } catch (Exception e) { 880 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 881 } 882 } 883 } 884 def startedByUser = ""; 885 def causes = currentBuild.getBuildCauses() 886 echo "causes: " + causes.toString() 887 for (cause in causes) { 888 def causeString = cause.toString() 889 echo "current cause: " + causeString 890 def causeInfo = readJSON text: causeString 891 if (causeInfo.userId != null) { 892 startedByUser = causeInfo.userId 893 } 894 } 895 896 if (startedByUser.length() > 0) { 897 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 898 def author = trimIfGithubNoreplyUser(startedByUser) 899 echo "DEBUG: author: ${startedByUser}, ${author}" 900 if (userMappings.containsKey(author)) { 901 def slackUser = userMappings.get(author) 902 if (!suspectList.contains(slackUser)) { 903 echo "Added ${slackUser} as suspect" 904 retValue += " ${slackUser}" 905 suspectList.add(slackUser) 906 } 907 } else { 908 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 909 if (!suspectList.contains(author)) { 910 echo "Added ${author} as suspect" 911 retValue += " ${author}" 912 suspectList.add(author) 913 } 914 } 915 } else { 916 echo "Build not started by a user, not adding to notification list" 917 } 918 echo "returning suspect list: ${retValue}" 919 return retValue 920 } 921 922 @NonCPS 923 List extractReleaseTags(final String fileContent) { 924 List releases = [] 925 fileContent.eachLine { tag -> 926 releases << tag 927 } 928 return releases 929 } 930 931 // Called in Stage Clean workspace and checkout steps 932 @NonCPS 933 def getCommitList() { 934 echo "Checking for change sets" 935 def commitList = [] 936 def changeSets = currentBuild.changeSets 937 for (int i = 0; i < changeSets.size(); i++) { 938 echo "get commits from change set" 939 def commits = changeSets[i].items 940 for (int j = 0; j < commits.length; j++) { 941 def commit = commits[j] 942 def id = commit.commitId 943 echo "Add commit id: ${id}" 944 commitList.add(id) 945 } 946 } 947 return commitList 948 } 949 950 def getLatestReleaseVersion() { 951 final String releaseTags = readFile(file: "${workspace}/tags.txt") 952 list gitTags = extractReleaseTags(releaseTags) 953 echo "gitTags = ${gitTags}" 954 return gitTags.pop() 955 } 956 957 // Create a html file containing the links to the Verrazzano distributions 958 def captureDistributionURLs() { 959 script { 960 def BRANCH_IN_OS = "${env.BRANCH_NAME.replace("/", "%252F")}" 961 BASE_URL="https://objectstorage.${OCI_OS_REGION}.oraclecloud.com/n/${OCI_OS_NAMESPACE}/b/${OCI_OS_BUCKET}/o/${BRANCH_IN_OS}-last-clean-periodic-test" 962 LITE_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 963 FULL_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 964 sh """ 965 cat <<EOF > ${WORKSPACE}/${verrazzanoDistributionsFile} 966 <!DOCTYPE html> 967 <html> 968 <body> 969 <b>Verrazzano Release Distributions</b> 970 <ul> 971 <li><a href="${LITE_BUNDLE}">Verrazzano Lite Distribution</a></li> 972 <li><a href="${FULL_BUNDLE}">Verrazzano Full Distribution</a></li> 973 </ul> 974 </body> 975 </html> 976 EOF 977 """ 978 } 979 }