github.com/verrazzano/verrazzano@v1.7.1/ci/oke-ocidns/JenkinsfileDnsSweepTests (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 import groovy.transform.Field 5 6 @Field 7 def GIT_COMMIT_TO_USE = "" 8 @Field 9 def LAST_CLEAN_PERIODIC_COMMIT = "" 10 @Field 11 def VERRAZZANO_DEV_VERSION = "" 12 @Field 13 def RELEASABLE_IMAGES_OBJECT_STORE = "" 14 @Field 15 def TESTS_FAILED = false 16 @Field 17 def storeLocation="" 18 @Field 19 def verrazzanoPrefix="verrazzano-" 20 @Field 21 def fullBundle="" 22 @Field 23 def liteBundle="" 24 @Field 25 def SUSPECT_LIST = "" 26 @Field 27 def COMPARISON_URL_ON_FAILURE = "" 28 29 // The job name from which the verrazzano_images file is available to be copied to this job 30 // We will copy over and make it part of the artifacts of the periodic job, available when we want to release a candidate 31 @Field 32 def verrazzanoImagesJobProjectName = "verrazzano-examples" 33 @Field 34 def verrazzanoImagesFile = "verrazzano_images.txt" 35 @Field 36 def verrazzanoImagesBuildNumber = 0 // will be set to actual build number when the job is run 37 @Field 38 def periodicsUpToDate = false // If true, indicates that the periodics already passed at the latest commit 39 40 // Non Fields 41 def branchSpecificSchedule = getCronSchedule() 42 43 // File containing the links to download the Verrazzano distributions 44 @Field 45 def verrazzanoDistributionsFile = "verrazzano_distributions.html" 46 47 pipeline { 48 options { 49 skipDefaultCheckout true 50 disableConcurrentBuilds() 51 timestamps () 52 } 53 54 agent { 55 docker { 56 image "${RUNNER_DOCKER_IMAGE}" 57 args "${RUNNER_DOCKER_ARGS}" 58 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 59 registryCredentialsId 'ocir-pull-and-push-account' 60 label "pipeline-job-large" 61 } 62 } 63 64 65 parameters { 66 string (name: 'GIT_COMMIT_TO_USE', 67 defaultValue: 'NONE', 68 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 69 trim: true) 70 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 71 defaultValue: 'NONE', 72 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 73 trim: true) 74 string (name: 'TAGGED_TESTS', 75 defaultValue: '', 76 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 77 trim: true) 78 string (name: 'INCLUDED_TESTS', 79 defaultValue: '.*', 80 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 81 trim: true) 82 string (name: 'EXCLUDED_TESTS', 83 defaultValue: '_excluded_test', 84 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 85 trim: true) 86 booleanParam (description: 'Force execution of the tests even if up-to-date', name: 'FORCE', defaultValue: false) 87 booleanParam (description: 'Skip test execution (for debugging)', name: 'DRY_RUN', defaultValue: false) 88 booleanParam (description: 'Run KinD Based DNS tests ', name: 'KIND_DNS_TESTS', defaultValue: true) 89 booleanParam (description: 'Run OKE Based DNS tests ', name: 'OKE_DNS_TESTS', defaultValue: false) 90 } 91 92 environment { 93 OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment') 94 OCIR_SCAN_TARGET = credentials('ocir-scan-target') 95 OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry') 96 OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path') 97 DOCKER_SCAN_CREDS = credentials('v8odev-ocir') 98 DOCKER_CREDS = credentials('github-packages-credentials-rw') 99 DOCKER_REPO = 'ghcr.io' 100 101 OCI_CLI_AUTH="instance_principal" 102 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 103 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 104 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 105 106 STABLE_COMMIT_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-stable-commit.txt" 107 CLEAN_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test/verrazzano_periodic-commit.txt" 108 109 STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt" 110 CLEAN_PERIODIC_LOCATION = "${WORKSPACE}/last-clean-periodic-commit.txt" 111 112 OCI_OS_REGION="us-phoenix-1" 113 } 114 115 // This job runs against the latest stable master commit. That is defined as the last clean master build and test run whose 116 // commit has been stored in object storage. This job will fetch that commit from master and run extended tests using that. 117 // This job is NOT currently setup to run extended tests from other branches, if you need to run those extended jobs you will 118 // need to run those against your branch individually. 119 120 stages { 121 stage('Clean workspace and checkout') { 122 steps { 123 sh """ 124 echo "${NODE_LABELS}" 125 """ 126 script { 127 if (params.GIT_COMMIT_TO_USE == "NONE") { 128 echo "Specific GIT commit was not specified, use current head" 129 def scmInfo = checkout([ 130 $class: 'GitSCM', 131 branches: [[name: env.BRANCH_NAME]], 132 doGenerateSubmoduleConfigurations: false, 133 extensions: [], 134 submoduleCfg: [], 135 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 136 env.GIT_COMMIT = scmInfo.GIT_COMMIT 137 env.GIT_BRANCH = scmInfo.GIT_BRANCH 138 } else { 139 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 140 def scmInfo = checkout([ 141 $class: 'GitSCM', 142 branches: [[name: params.GIT_COMMIT_TO_USE]], 143 doGenerateSubmoduleConfigurations: false, 144 extensions: [], 145 submoduleCfg: [], 146 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 147 env.GIT_COMMIT = scmInfo.GIT_COMMIT 148 env.GIT_BRANCH = scmInfo.GIT_BRANCH 149 // If the commit we were handed is not what the SCM says we are using, fail 150 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 151 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 152 exit 1 153 } 154 } 155 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 156 } 157 158 script { 159 def props = readProperties file: '.verrazzano-development-version' 160 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 161 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 162 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 163 // update the description with some meaningful info 164 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 165 def currentCommitHash = env.GIT_COMMIT 166 def commitList = getCommitList() 167 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 168 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 169 SUSPECT_LIST = getSuspectList(commitList, userMappings) 170 echo "Suspect list: ${SUSPECT_LIST}" 171 } 172 } 173 } 174 } 175 176 stage ('All DNS Tests with KinD') { 177 when { 178 allOf { 179 expression { return runKindDnsTests() } 180 } 181 } 182 parallel { 183 stage('oci user principal with global dns') { 184 steps { 185 retry(count: JOB_PROMOTION_RETRIES) { 186 script { 187 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 188 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 189 parameters: [ 190 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 191 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 192 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 193 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 194 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 195 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 196 ], wait: true 197 } 198 } 199 } 200 } 201 202 stage('oci instance principal with global dns') { 203 steps { 204 retry(count: JOB_PROMOTION_RETRIES) { 205 script { 206 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 207 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 208 parameters: [ 209 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 210 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 211 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 212 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 213 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 214 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 215 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 216 ], wait: true 217 } 218 } 219 } 220 } 221 222 stage('oci user principal with private dns') { 223 steps { 224 retry(count: JOB_PROMOTION_RETRIES) { 225 script { 226 227 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 228 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 229 parameters: [ 230 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 231 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 232 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 233 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 234 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 235 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 236 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 237 ], wait: true 238 } 239 } 240 } 241 } 242 243 stage('oci instance principal with private dns') { 244 steps { 245 retry(count: JOB_PROMOTION_RETRIES) { 246 script { 247 248 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=GLOBAL") 249 build job: "/verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 250 parameters: [ 251 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 252 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 253 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 254 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 255 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 256 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 257 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 258 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 259 ], wait: true 260 } 261 } 262 } 263 } 264 265 stage('oci user principal with global dns cert-issuer acme and env as staging') { 266 steps { 267 retry(count: JOB_PROMOTION_RETRIES) { 268 script { 269 build job: "verrazzano-oci-dns-kind-tests/${CLEAN_BRANCH_NAME}", 270 parameters: [ 271 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 272 string(name: 'CERT_ISSUER', value: "acme"), 273 string(name: 'ACME_ENVIRONMENT', value: "staging"), 274 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: false), 275 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 276 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 277 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 278 ], wait: true 279 } 280 } 281 } 282 post { 283 failure { 284 script { 285 TESTS_FAILED = true 286 } 287 } 288 } 289 } 290 291 292 } 293 } 294 295 296 297 298 stage ('All DNS Tests with OKE') { 299 when { 300 allOf { 301 expression { return runOkeDnsTests() } 302 } 303 } 304 parallel { 305 306 stage('oci user principal with global dns and private istio lb') { 307 steps { 308 retry(count: JOB_PROMOTION_RETRIES) { 309 script { 310 311 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 312 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 313 parameters: [ 314 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 315 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 316 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 317 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 318 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 319 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 320 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 321 ], wait: true 322 } 323 } 324 } 325 } 326 327 stage('oci user principal with global dns and private nginx lb') { 328 steps { 329 retry(count: JOB_PROMOTION_RETRIES) { 330 script { 331 332 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 333 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 334 parameters: [ 335 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 336 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 337 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 338 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 339 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 340 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 341 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 342 ], wait: true 343 } 344 } 345 } 346 } 347 348 stage('oci user principal with global dns and private istio and nginx lb') { 349 steps { 350 retry(count: JOB_PROMOTION_RETRIES) { 351 script { 352 353 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 354 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 355 parameters: [ 356 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 357 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 358 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 359 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 360 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 361 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 362 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 363 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 364 ], wait: true 365 } 366 } 367 } 368 } 369 370 371 372 stage('oci instance principal with global dns and private istio lb') { 373 steps { 374 retry(count: JOB_PROMOTION_RETRIES) { 375 script { 376 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 377 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 378 parameters: [ 379 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 380 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 381 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 382 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 383 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 384 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 385 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 386 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 387 ], wait: true 388 } 389 } 390 } 391 } 392 393 stage('oci instance principal with global dns and private nginx lb') { 394 steps { 395 retry(count: JOB_PROMOTION_RETRIES) { 396 script { 397 398 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 399 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 400 parameters: [ 401 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 402 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 403 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 404 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 405 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 406 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 407 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 408 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 409 ], wait: true 410 } 411 } 412 } 413 } 414 415 stage('oci instance principal with global dns and private istio and nginx lb') { 416 steps { 417 retry(count: JOB_PROMOTION_RETRIES) { 418 script { 419 420 println("AUTH_TYPE=IP : DNS_SCOPE=GLOBAL : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 421 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 422 parameters: [ 423 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 424 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 425 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 426 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 427 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 428 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 429 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 430 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 431 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 432 ], wait: true 433 } 434 } 435 } 436 } 437 438 439 440 441 442 443 stage('oci user principal with private dns and private istio lb') { 444 steps { 445 retry(count: JOB_PROMOTION_RETRIES) { 446 script { 447 448 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 449 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 450 parameters: [ 451 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 452 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 453 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 454 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 455 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 456 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 457 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 458 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 459 ], wait: true 460 } 461 } 462 } 463 } 464 465 stage('oci user principal with private dns and private nginx lb') { 466 steps { 467 retry(count: JOB_PROMOTION_RETRIES) { 468 script { 469 470 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 471 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 472 parameters: [ 473 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 474 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 475 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 476 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 477 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 478 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 479 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 480 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 481 ], wait: true 482 } 483 } 484 } 485 } 486 487 stage('oci user principal with private dns and private istio and nginx lb') { 488 steps { 489 retry(count: JOB_PROMOTION_RETRIES) { 490 script { 491 println("AUTH_TYPE=UserPrincipal : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 492 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 493 parameters: [ 494 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 495 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 496 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 497 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 498 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 499 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 500 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 501 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 502 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 503 ], wait: true 504 } 505 } 506 } 507 } 508 509 510 511 stage('oci instance principal with private dns and private istio lb') { 512 steps { 513 retry(count: JOB_PROMOTION_RETRIES) { 514 script { 515 516 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=GLOBAL : Istio_LB=PRIVATE") 517 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 518 parameters: [ 519 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 520 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 521 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 522 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 523 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 524 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 525 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 526 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 527 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 528 ], wait: true 529 } 530 } 531 } 532 } 533 534 stage('oci instance principal with private dns and private nginx lb') { 535 steps { 536 retry(count: JOB_PROMOTION_RETRIES) { 537 script { 538 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=GLOBAL") 539 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 540 parameters: [ 541 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 542 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 543 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 544 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 545 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 546 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 547 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 548 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 549 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 550 ], wait: true 551 } 552 } 553 } 554 } 555 556 stage('oci instance principal with private dns and private istio and nginx lb') { 557 steps { 558 retry(count: JOB_PROMOTION_RETRIES) { 559 script { 560 println("AUTH_TYPE=IP : DNS_SCOPE=PRIVATE : Nginx_LB=PRIVATE : Istio_LB=PRIVATE") 561 build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}", 562 parameters: [ 563 string(name: 'OCI_DNS_AUTH', value: 'instance_principal'), 564 string(name: 'DNS_SCOPE', value: 'PRIVATE'), 565 string(name: 'ISTIO_LB_SCOPE', value: 'PRIVATE'), 566 string(name: 'NGINX_LB_SCOPE', value: 'PRIVATE'), 567 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 568 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 569 booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true), 570 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 571 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 572 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 573 ], wait: true 574 } 575 } 576 } 577 } 578 } 579 } 580 } 581 post { 582 always { 583 script { 584 if (verrazzanoImagesBuildNumber > 0) { 585 copyArtifacts(projectName: "${verrazzanoImagesJobProjectName}/${CLEAN_BRANCH_NAME}", 586 selector: specific("${verrazzanoImagesBuildNumber}"), 587 filter: verrazzanoImagesFile) 588 sh """ 589 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BRANCH_NAME}/${RELEASABLE_IMAGES_OBJECT_STORE} --file ${verrazzanoImagesFile} 590 """ 591 } else { 592 println("Cannot copy ${verrazzanoImagesFile} from the project ${verrazzanoImagesJobProjectName} - no build number is available to copy from.") 593 } 594 } 595 archiveArtifacts artifacts: "**/prerelease_validation.out,**/release_status.out,**/${verrazzanoImagesFile},**/${verrazzanoDistributionsFile}", allowEmptyArchive: true 596 } 597 failure { 598 script { 599 if (isAlertingEnabled()) { 600 if (isPagerDutyEnabled()) { 601 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", 602 incDescription: "Verrazzano Periodic Tests: ${env.JOB_NAME} - Failed", 603 incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 604 } 605 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}\n\nChange comparison: ${COMPARISON_URL_ON_FAILURE}" ) 606 echo "done alerts" 607 } 608 } 609 } 610 cleanup { 611 deleteDir() 612 } 613 } 614 } 615 616 // Preliminary job checks and display updates 617 def preliminaryChecks() { 618 // Get the last stable commit ID to pass the triggered tests 619 def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}" 620 GIT_COMMIT_TO_USE = stableCommitProps['git-commit'] 621 echo "Last stable commit: ${GIT_COMMIT_TO_USE}" 622 623 LAST_CLEAN_PERIODIC_COMMIT=getLastCleanPeriodicCommit() 624 echo "Last clean periodics commit: ${LAST_CLEAN_PERIODIC_COMMIT}" 625 626 if (LAST_CLEAN_PERIODIC_COMMIT == GIT_COMMIT_TO_USE) { 627 periodicsUpToDate = true 628 } 629 630 echo "Up to date: ${periodicsUpToDate}" 631 echo "Dry run: ${params.DRY_RUN}" 632 echo "Force run: ${params.FORCE}" 633 echo "Execute tests: " + runTests() 634 635 // Indicate in title if run is up-to-date or dry-run 636 if (params.DRY_RUN) { 637 currentBuild.displayName = "${currentBuild.displayName} : DRY-RUN" 638 } 639 if (periodicsUpToDate) { 640 currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE" 641 } 642 if (params.FORCE) { 643 currentBuild.displayName = "${currentBuild.displayName} : FORCE" 644 } 645 646 if (runTests()) { 647 echo "Executing periodic tests for commit ${GIT_COMMIT_TO_USE}" 648 } 649 } 650 651 def dockerLogins() { 652 try { 653 sh """ 654 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 655 """ 656 } catch(error) { 657 echo "docker login failed, retrying after sleep" 658 retry(4) { 659 sleep(30) 660 sh """ 661 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 662 """ 663 } 664 } 665 if (!(env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-1."))) { 666 try { 667 sh """ 668 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 669 """ 670 } catch(error) { 671 echo "docker login failed, retrying after sleep" 672 retry(4) { 673 sleep(30) 674 sh """ 675 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 676 """ 677 } 678 } 679 } 680 } 681 682 def scmCheckout() { 683 echo "${NODE_LABELS}" 684 echo "SCM checkout of ${GIT_COMMIT_TO_USE}" 685 def scmInfo = checkout([ 686 $class: 'GitSCM', 687 branches: [[name: GIT_COMMIT_TO_USE]], 688 doGenerateSubmoduleConfigurations: false, 689 extensions: [], 690 submoduleCfg: [], 691 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 692 env.GIT_COMMIT = scmInfo.GIT_COMMIT 693 env.GIT_BRANCH = scmInfo.GIT_BRANCH 694 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 695 // If the commit we were handed is not what the SCM says we are using, fail 696 if (!env.GIT_COMMIT.equals(GIT_COMMIT_TO_USE)) { 697 error( "SCM didn't checkout the commit we expected. Expected: ${GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}") 698 } 699 700 if (LAST_CLEAN_PERIODIC_COMMIT != null) { 701 COMPARISON_URL_ON_FAILURE = "https://github.com/verrazzano/verrazzano/compare/${LAST_CLEAN_PERIODIC_COMMIT}...${GIT_COMMIT_TO_USE}" 702 def lastClean = "${LAST_CLEAN_PERIODIC_COMMIT}" 703 def currentStable = "${GIT_COMMIT_TO_USE}" 704 def commitList = getCommitListFromGitLog(lastClean, currentStable) 705 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 706 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 707 SUSPECT_LIST = getSuspectList(commitList, userMappings) 708 echo "Suspect list: ${SUSPECT_LIST}" 709 } 710 } 711 echo "URL if fails: ${COMPARISON_URL_ON_FAILURE}" 712 } 713 714 def cleanWorkspaceAndCheckout() { 715 scmCheckout() 716 dockerLogins() 717 def props = readProperties file: '.verrazzano-development-version' 718 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 719 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 720 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 721 // update the description with some meaningful info 722 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + GIT_COMMIT_TO_USE 723 storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 724 fullBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 725 liteBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 726 RELEASABLE_IMAGES_OBJECT_STORE = "verrazzano_${VERRAZZANO_DEV_VERSION}-images.txt" 727 } 728 729 // Returns the last clean commit for the periodics, or null if the commit file does not exist yet. 730 // - fails the pipeline if any error other than 404 is returned by the OCI CLI 731 def getLastCleanPeriodicCommit() { 732 lastPeriodicCommitCommandOutput = sh ( 733 label: "Get last clean periodic commit ID", 734 script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_PERIODIC_OS_LOCATION} --file ${CLEAN_PERIODIC_LOCATION} 2>&1 || true", 735 returnStdout: true 736 ).trim() 737 echo "command out: ${lastPeriodicCommitCommandOutput}" 738 if (lastPeriodicCommitCommandOutput.length() > 0) { 739 // We can get warning messages here as well even when the command succeeded, so be more precise on the checking 740 if (lastPeriodicCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) { 741 // If we think we had a status: NNN, we ignore 404 and fail for others 742 assert lastPeriodicCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 743 } else { 744 // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error 745 sh """ 746 if [ ! -f ${CLEAN_PERIODIC_LOCATION} ]; then 747 echo "An unexpected error occurred getting last periodic commit from ObjectStore: ${lastPeriodicCommitCommandOutput}" 748 exit 1 749 fi 750 """ 751 } 752 } 753 // Get the commit ID for the last known clean pass of the Periodic tests 754 def cleanPeriodicsCommitProps = readProperties file: "${CLEAN_PERIODIC_LOCATION}" 755 return cleanPeriodicsCommitProps['git-commit'] 756 } 757 758 // Checks all the conditions gating test execution and coallates the result 759 def runTests() { 760 return params.FORCE || ( ! periodicsUpToDate && ! params.DRY_RUN ) 761 } 762 763 def runKindDnsTests() { 764 return params.KIND_DNS_TESTS && ( ! params.DRY_RUN ) 765 } 766 767 def runOkeDnsTests() { 768 return params.OKE_DNS_TESTS && ( ! params.DRY_RUN ) 769 } 770 771 def isAlertingEnabled() { 772 // this controls whether any alerting happens for these tests 773 if (NOTIFY_PERIODIC_FAILURES.equals("true") && (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-1."))) { 774 echo "Alert notifications enabled for ${env.BRANCH_NAME}" 775 return true 776 } 777 return false 778 } 779 780 def releaseValidationChecks() { 781 def built = build job: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 782 parameters: [ 783 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT) 784 ], wait: true, propagate: false 785 println("Result of verrazzano-prerelease-check is ${built.result}") 786 dir ("${WORKSPACE}") { 787 copyArtifacts(projectName: "verrazzano-prerelease-check/${CLEAN_BRANCH_NAME}", 788 selector: specific("${built.number}")); 789 def releaseStatus = readFile file: "release_status.out" 790 currentBuild.displayName = "${currentBuild.displayName} : ${releaseStatus}" 791 } 792 } 793 794 def isPagerDutyEnabled() { 795 // this additionally controls whether PD alerts are enabled (note that you must also enable alerting in general as well if you want these) 796 if (NOTIFY_PAGERDUTY_PERIODIC_FAILURES.equals("true")) { 797 echo "Pager-Duty notifications enabled via global override setting" 798 return true 799 } 800 return false 801 } 802 803 def getCronSchedule() { 804 if (env.BRANCH_NAME.equals("master")) { 805 return "H */6 * * *" 806 } else if (env.BRANCH_NAME.startsWith("release-1")) { 807 return "@daily" 808 } 809 return "" 810 } 811 812 // Called in Stage Clean workspace and checkout steps 813 def getCommitListFromGitLog(lastClean, currentStable) { 814 echo "Checking for change sets" 815 def commitList = sh(returnStdout: true, script: "git log ${lastClean}...${currentStable} --oneline | cut -d \" \" -f 1").trim().split('\n') 816 for (int i = 0; i < commitList.size(); i++) { 817 echo "Found commit id: ${commitList[i]}" 818 } 819 return commitList 820 } 821 822 def trimIfGithubNoreplyUser(userIn) { 823 if (userIn == null) { 824 echo "Not a github noreply user, not trimming: ${userIn}" 825 return userIn 826 } 827 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 828 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 829 return userOut; 830 } 831 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 832 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 833 return userOut; 834 } 835 if (userIn.matches(".*@users.noreply.github.com")) { 836 def userOut = userIn.substring(0, userIn.indexOf("@")) 837 return userOut; 838 } 839 echo "Not a github noreply user, not trimming: ${userIn}" 840 return userIn 841 } 842 843 def getSuspectList(commitList, userMappings) { 844 def retValue = "" 845 def suspectList = [] 846 if (commitList == null || commitList.size() == 0) { 847 echo "No commits to form suspect list" 848 } else { 849 for (int i = 0; i < commitList.size(); i++) { 850 def id = commitList[i] 851 try { 852 def gitAuthor = sh( 853 script: "git log --format='%ae' '$id^!'", 854 returnStdout: true 855 ).trim() 856 if (gitAuthor != null) { 857 def author = trimIfGithubNoreplyUser(gitAuthor) 858 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 859 if (userMappings.containsKey(author)) { 860 def slackUser = userMappings.get(author) 861 if (!suspectList.contains(slackUser)) { 862 echo "Added ${slackUser} as suspect" 863 retValue += " ${slackUser}" 864 suspectList.add(slackUser) 865 } 866 } else { 867 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 868 if (!suspectList.contains(author)) { 869 echo "Added ${author} as suspect" 870 retValue += " ${author}" 871 suspectList.add(author) 872 } 873 } 874 } else { 875 echo "No author returned from git" 876 } 877 } catch (Exception e) { 878 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 879 } 880 } 881 } 882 def startedByUser = ""; 883 def causes = currentBuild.getBuildCauses() 884 echo "causes: " + causes.toString() 885 for (cause in causes) { 886 def causeString = cause.toString() 887 echo "current cause: " + causeString 888 def causeInfo = readJSON text: causeString 889 if (causeInfo.userId != null) { 890 startedByUser = causeInfo.userId 891 } 892 } 893 894 if (startedByUser.length() > 0) { 895 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 896 def author = trimIfGithubNoreplyUser(startedByUser) 897 echo "DEBUG: author: ${startedByUser}, ${author}" 898 if (userMappings.containsKey(author)) { 899 def slackUser = userMappings.get(author) 900 if (!suspectList.contains(slackUser)) { 901 echo "Added ${slackUser} as suspect" 902 retValue += " ${slackUser}" 903 suspectList.add(slackUser) 904 } 905 } else { 906 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 907 if (!suspectList.contains(author)) { 908 echo "Added ${author} as suspect" 909 retValue += " ${author}" 910 suspectList.add(author) 911 } 912 } 913 } else { 914 echo "Build not started by a user, not adding to notification list" 915 } 916 echo "returning suspect list: ${retValue}" 917 return retValue 918 } 919 920 @NonCPS 921 List extractReleaseTags(final String fileContent) { 922 List releases = [] 923 fileContent.eachLine { tag -> 924 releases << tag 925 } 926 return releases 927 } 928 929 // Called in Stage Clean workspace and checkout steps 930 @NonCPS 931 def getCommitList() { 932 echo "Checking for change sets" 933 def commitList = [] 934 def changeSets = currentBuild.changeSets 935 for (int i = 0; i < changeSets.size(); i++) { 936 echo "get commits from change set" 937 def commits = changeSets[i].items 938 for (int j = 0; j < commits.length; j++) { 939 def commit = commits[j] 940 def id = commit.commitId 941 echo "Add commit id: ${id}" 942 commitList.add(id) 943 } 944 } 945 return commitList 946 } 947 948 def getLatestReleaseVersion() { 949 final String releaseTags = readFile(file: "${workspace}/tags.txt") 950 list gitTags = extractReleaseTags(releaseTags) 951 echo "gitTags = ${gitTags}" 952 return gitTags.pop() 953 } 954 955 // Create a html file containing the links to the Verrazzano distributions 956 def captureDistributionURLs() { 957 script { 958 def BRANCH_IN_OS = "${env.BRANCH_NAME.replace("/", "%252F")}" 959 BASE_URL="https://objectstorage.${OCI_OS_REGION}.oraclecloud.com/n/${OCI_OS_NAMESPACE}/b/${OCI_OS_BUCKET}/o/${BRANCH_IN_OS}-last-clean-periodic-test" 960 LITE_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 961 FULL_BUNDLE="${BASE_URL}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 962 sh """ 963 cat <<EOF > ${WORKSPACE}/${verrazzanoDistributionsFile} 964 <!DOCTYPE html> 965 <html> 966 <body> 967 <b>Verrazzano Release Distributions</b> 968 <ul> 969 <li><a href="${LITE_BUNDLE}">Verrazzano Lite Distribution</a></li> 970 <li><a href="${FULL_BUNDLE}">Verrazzano Full Distribution</a></li> 971 </ul> 972 </body> 973 </html> 974 EOF 975 """ 976 } 977 }