github.com/verrazzano/verrazzano@v1.7.1/ci/upgrade-paths/Jenkinsfile (about) 1 // Copyright (c) 2023, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def SKIP_ACCEPTANCE_TESTS = false 6 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 8 def acmeEnvironments = [ "staging", "production" ] 9 Collections.shuffle(availableRegions) 10 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 11 def dns_zone_ocid = 'dummy' 12 def OKE_CLUSTER_PREFIX = "" 13 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 14 15 def KUBECTL = "kubectl" 16 def VZ_CLI = "vz cli" 17 def installationMethodsPreupgrade = [ KUBECTL, VZ_CLI ] 18 Collections.shuffle(installationMethodsPreupgrade) 19 def installationMethodsUpgrade = [ KUBECTL, VZ_CLI ] 20 Collections.shuffle(installationMethodsUpgrade) 21 22 installerFileName = "install-verrazzano.yaml" 23 24 pipeline { 25 options { 26 skipDefaultCheckout true 27 timestamps () 28 } 29 30 agent { 31 docker { 32 image "${RUNNER_DOCKER_IMAGE}" 33 args "${RUNNER_DOCKER_ARGS}" 34 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 35 registryCredentialsId 'ocir-pull-and-push-account' 36 label "${agentLabel}" 37 } 38 } 39 parameters { 40 booleanParam (description: 'Whether to use External Elasticsearch', name: 'EXTERNAL_ELASTICSEARCH', defaultValue: false) 41 choice (description: 'Number of Cluster', name: 'TOTAL_CLUSTERS', choices: ["2", "1", "3"]) 42 choice (description: 'Predefined config permutations for Verrazzano installation. Prod profile is the default profile for NONE', name: 'VZ_INSTALL_CONFIG', 43 choices: ["NONE", "dev-kind-persistence"]) 44 choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV', 45 choices: ["KIND", "magicdns_oke", "ocidns_oke"]) 46 choice (description: 'ACME Certificate Environment (Staging or Production)', name: 'ACME_ENVIRONMENT', 47 choices: acmeEnvironments) 48 choice (description: 'OCI region to launch OKE clusters', name: 'OKE_CLUSTER_REGION', 49 // 1st choice is the default value 50 choices: availableRegions ) 51 choice (description: 'Verrazzano pre-upgrade installation using either kubectl or vz cli ', name: 'INSTALLATION_METHOD_PREUPGRADE', 52 // 1st choice is the default value 53 choices: installationMethodsPreupgrade ) 54 choice (description: 'Verrazzano upgrade installation using either kubectl or vz cli ', name: 'INSTALLATION_METHOD_UPGRADE', 55 // 1st choice is the default value 56 choices: installationMethodsUpgrade ) 57 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 58 // 1st choice is the default value 59 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2" ]) 60 choice (name: 'OKE_CLUSTER_VERSION', 61 description: 'Kubernetes Version for OKE Cluster', 62 // 1st choice is the default value 63 choices: [ "v1.24.1", "v1.25.4", "v1.26.2", "v1.27.2" ]) 64 choice (name: 'KIND_CLUSTER_VERSION', 65 description: 'Kubernetes Version for KIND Cluster', 66 // 1st choice is the default value 67 choices: [ "1.24", "1.25", "1.26", "1.27" ]) 68 string (name: 'VERSION_FOR_INSTALL', 69 defaultValue: 'v1.4.0', 70 description: 'This is the Verrazzano version for install before doing an upgrade. By default, the v1.4.0 release will be installed', 71 trim: true) 72 string (name: 'VERSION_FOR_UPGRADE', 73 defaultValue: 'NONE', 74 description: 'This is the Verrazzano version for Upgrade. By default, the Verrazzano tip will be used.', 75 trim: true) 76 choice (name: 'IS_TRIGGERED_MANUALLY', 77 description: 'This flag is used to test upgrade within existing releases. the default value is set to YES to enable users to trigger customized version upgrades. Default: YES', 78 choices: [ "YES", "NO" ]) 79 string (name: 'GIT_COMMIT_TO_USE', 80 defaultValue: 'NONE', 81 description: 'This is the full git commit hash from the source build to be used for all jobs', 82 trim: true) 83 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 84 defaultValue: 'NONE', 85 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the latest operator.yaml published to the Verrazzano Object Store will be used', 86 trim: true) 87 choice (name: 'ADMIN_CLUSTER_PROFILE', 88 description: 'Verrazzano Admin Cluster install profile name', 89 // 1st choice is the default value 90 choices: [ "prod", "dev" ]) 91 choice (name: 'MANAGED_CLUSTER_PROFILE', 92 description: 'Verrazzano Managed Cluster install profile name', 93 // 1st choice is the default value 94 choices: [ "managed-cluster", "prod", "dev" ]) 95 choice (name: 'WILDCARD_DNS_DOMAIN', 96 description: 'This is the wildcard DNS domain', 97 // 1st choice is the default value 98 choices: [ "nip.io", "sslip.io"]) 99 choice (name: 'CRD_API_VERSION', 100 description: 'This is the API crd version.', 101 // 1st choice is the default value 102 choices: [ "v1alpha1", "v1beta1"]) 103 booleanParam (description: 'Whether to create the cluster with Calico for AT testing', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 104 string (name: 'CONSOLE_REPO_BRANCH', 105 defaultValue: '', 106 description: 'The branch to check out after cloning the console repository.', 107 trim: true) 108 string (name: 'TAGGED_TESTS', 109 defaultValue: '', 110 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 111 trim: true) 112 string (name: 'INCLUDED_TESTS', 113 defaultValue: '.*', 114 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 115 trim: true) 116 string (name: 'EXCLUDED_TESTS', 117 defaultValue: '_excluded_test', 118 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 119 trim: true) 120 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 121 } 122 123 environment { 124 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 125 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 126 DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}" 127 DOCKER_OAM_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins' 128 DOCKER_OAM_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator' 129 DOCKER_OAM_IMAGE_NAME = "${env.BRANCH_NAME == 'master' ? env.DOCKER_OAM_PUBLISH_IMAGE_NAME : env.DOCKER_OAM_CI_IMAGE_NAME}" 130 CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}" 131 GOPATH = '/home/opc/go' 132 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 133 DOCKER_CREDS = credentials('github-packages-credentials-rw') 134 DOCKER_EMAIL = credentials('github-packages-email') 135 DOCKER_REPO = 'ghcr.io' 136 DOCKER_NAMESPACE = 'verrazzano' 137 NETRC_FILE = credentials('netrc') 138 CLUSTER_NAME_PREFIX = 'verrazzano' 139 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 140 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 141 KUBECONFIG_DIR = "${WORKSPACE}/kubeconfig" 142 143 OCR_CREDS = credentials('ocr-pull-and-push-account') 144 OCR_REPO = 'container-registry.oracle.com' 145 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 146 147 TEST_ENV = "${params.TEST_ENV}" 148 MANAGED_CLUSTER_PROFILE = "${params.MANAGED_CLUSTER_PROFILE}" 149 ADMIN_CLUSTER_PROFILE = "${params.ADMIN_CLUSTER_PROFILE}" 150 151 // Find a better way to handle this 152 // OKE_CLUSTER_VERSION = "${params.KUBERNETES_VERSION == '1.17' ? 'v1.17.13' : 'v1.18.10'}" 153 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 154 TF_VAR_tenancy_id = credentials('oci-tenancy') 155 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 156 TF_VAR_user_id = credentials('oci-user-ocid') 157 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 158 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 159 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 160 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 161 TF_VAR_api_private_key_path = credentials('oci-api-key') 162 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 163 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 164 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 165 166 OCI_CLI_TENANCY = credentials('oci-tenancy') 167 OCI_CLI_USER = credentials('oci-user-ocid') 168 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 169 OCI_CLI_KEY_FILE = credentials('oci-api-key') 170 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 171 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 172 173 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-vz-prod-kind-upgrade.yaml" 174 INSTALL_CONFIG_FILE_OCIDNS = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-ocidns.yaml" 175 INSTALL_CONFIG_FILE_NIPIO = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-nipio.yaml" 176 OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io" 177 ACME_ENVIRONMENT="${params.ACME_ENVIRONMENT}" 178 CREATE_EXTERNAL_OPENSEARCH = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create-external-os.sh" 179 180 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 181 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 182 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 183 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 184 ADMIN_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 185 186 // Environment variables required to capture cluster snapshot and bug report on test failure 187 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 188 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 189 CAPTURE_FULL_CLUSTER="false" 190 191 // Environment variable for Verrazzano CLI executable 192 VZ_COMMAND="${GO_REPO_PATH}/vz" 193 194 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 195 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 196 EXTERNAL_ELASTICSEARCH = "${params.EXTERNAL_ELASTICSEARCH}" 197 198 // used for console artifact capture on failure 199 JENKINS_READ = credentials('jenkins-auditor') 200 OCI_CLI_AUTH="instance_principal" 201 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 202 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 203 OCI_OS_LOCATION="${OCI_OS_LOCATION}" 204 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 205 206 // used to emit metrics 207 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 208 TEST_ENV_LABEL = "${params.TEST_ENV}" 209 210 // sample app deployed before upgrade and UI console tests 211 SAMPLE_APP_NAME="hello-helidon" 212 SAMPLE_APP_NAMESPACE="hello-helidon" 213 SAMPLE_APP_PROJECT="hello-helidon-sample-proj" 214 SAMPLE_APP_COMPONENT="hello-helidon-component" 215 216 // used to generate Ginkgo test reports 217 TEST_REPORT = "test-report.xml" 218 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 219 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 220 221 VERSION_FOR_UPGRADE_ENV = "${params.VERSION_FOR_UPGRADE}" 222 VERSION_FOR_INSTALL = "${params.VERSION_FOR_INSTALL}" 223 TARGET_UPGRADE_OPERATOR_YAML = "${WORKSPACE}/target-upgrade-operator.yaml" 224 RUNNING_OPERATOR_VERSION = "" 225 } 226 227 stages { 228 stage('Clean workspace and checkout') { 229 steps { 230 printNodeLabels() 231 232 performSourceCodeCheckout() 233 234 performNetRCOperations() 235 236 performDockerLogin() 237 238 performVerrazzanoDirectoryOperations() 239 240 setInitValues() 241 242 downloadCLI() 243 } 244 } 245 246 stage('Install and Configure') { 247 when { 248 allOf { 249 not { buildingTag() } 250 anyOf { 251 branch 'master'; 252 expression {SKIP_ACCEPTANCE_TESTS == false}; 253 } 254 } 255 } 256 stages { 257 stage('Prepare AT environment') { 258 parallel { 259 stage('Create Kind Clusters') { 260 when { expression { return params.TEST_ENV == 'KIND' } } 261 steps { 262 createKindClusters() 263 } 264 } 265 stage('Create OKE Clusters') { 266 when { expression { return params.TEST_ENV == 'ocidns_oke' || params.TEST_ENV == 'magicdns_oke'} } 267 steps { 268 echo "OKE Cluster Prefix: ${OKE_CLUSTER_PREFIX}" 269 createOKEClusters("${OKE_CLUSTER_PREFIX}") 270 } 271 } 272 } 273 } 274 stage("Configure Clusters") { 275 parallel { 276 stage("Configure OKE/OCI DNS") { 277 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 278 stages { 279 stage('Create OCI DNS zone') { 280 steps { 281 createOCIDNSZone() 282 } 283 } 284 stage('Configure OCI DNS Resources') { 285 environment { 286 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 287 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 288 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 289 } 290 steps { 291 createTestOCIConfigSecret() 292 } 293 } 294 stage('Configure OCI DNS Installers') { 295 environment { 296 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 297 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 298 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 299 } 300 steps { 301 script { 302 configureVerrazzanoInstallers(env.INSTALL_CONFIG_FILE_OCIDNS, "./tests/e2e/config/scripts/process_oci_dns_install_yaml.sh", "acme", params.ACME_ENVIRONMENT) 303 } 304 } 305 } 306 } 307 } 308 stage("Configure KinD") { 309 when { expression { return params.TEST_ENV == 'KIND' } } 310 steps { 311 configureVerrazzanoInstallers(INSTALL_CONFIG_FILE_KIND,"./tests/e2e/config/scripts/process_kind_install_yaml.sh", params.WILDCARD_DNS_DOMAIN) 312 } 313 } 314 stage("Configure OKE/MagicDNS") { 315 when { expression { return params.TEST_ENV == 'magicdns_oke' } } 316 steps { 317 configureVerrazzanoInstallers(env.INSTALL_CONFIG_FILE_NIPIO, "./tests/e2e/config/scripts/process_nipio_install_yaml.sh", params.WILDCARD_DNS_DOMAIN) 318 } 319 } 320 } 321 } 322 stage ('Install Verrazzano') { 323 steps { 324 script { 325 getVerrazzanoOperatorYaml() 326 } 327 installVerrazzano() 328 } 329 post { 330 always { 331 script { 332 dumpInstallLogs() 333 } 334 } 335 failure { 336 script { 337 dumpK8sCluster("${WORKSPACE}/install-failure-cluster-snapshot") 338 } 339 } 340 } 341 } 342 stage('Multicluster Registration') { 343 when { expression { isMultiCluster() } } 344 stages { 345 stage('Register managed clusters') { 346 steps { 347 registerManagedClusters() 348 } 349 } 350 stage('verify-register') { 351 steps { 352 verifyRegisterManagedClusters(true) 353 } 354 } 355 } 356 } 357 stage ('Pre-upgrade Acceptance Tests') { 358 parallel { 359 stage('pre-upgrade verify keycloak') { 360 steps { 361 runGinkgoRandomizeSerialAdmin('upgrade/pre-upgrade/keycloak', "${TEST_DUMP_ROOT}/keycloak") 362 } 363 } 364 stage('opensearch pre-upgrade') { 365 steps { 366 runGinkgoRandomizeSerialAdmin('upgrade/pre-upgrade/opensearch', "${TEST_DUMP_ROOT}/opensearch") 367 } 368 } 369 stage('opensearch-dashboards pre-upgrade') { 370 steps { 371 runGinkgoRandomizeSerialAdmin('upgrade/pre-upgrade/opensearch-dashboards', "${TEST_DUMP_ROOT}/opensearch-dashboards") 372 } 373 } 374 stage('Grafana pre-upgrade') { 375 steps { 376 runGinkgoRandomizeSerialAdmin('upgrade/pre-upgrade/grafana', "${TEST_DUMP_ROOT}/grafana") 377 } 378 } 379 stage('Metricsbinding pre-upgrade') { 380 steps { 381 runGinkgoRandomizeSerialAdmin('upgrade/pre-upgrade/metricsbinding', "${TEST_DUMP_ROOT}/metricsbinding") 382 } 383 } 384 stage ('examples helidon') { 385 when { expression { isVZOperatorMinimumCompatibleVersion("v1.1.0") } } 386 steps { 387 runGinkgoVerify('examples/helidon', "${TEST_DUMP_ROOT}/helidon-workload", "false", "true", "true", "hello-helidon-sc") 388 } 389 } 390 stage ('mc examples helidon') { 391 when { expression { isMultiCluster() && isVZOperatorMinimumCompatibleVersion("v1.1.0") } } 392 steps { 393 runGinkgoVerifyManaged("multicluster/examples/helidon", "${TEST_DUMP_ROOT}/mchelidon-workload", "false", "true", "true") 394 } 395 } 396 stage ('coherence workload') { 397 when { expression { isVZOperatorMinimumCompatibleVersion("v1.1.0") } } 398 steps { 399 runGinkgoVerify('workloads/coherence', "${TEST_DUMP_ROOT}/coherence-workload", "false", "false", "true", "hello-coherence-sc") 400 } 401 } 402 stage ('mc coherence workload') { 403 when { expression { isMultiCluster() && isVZOperatorMinimumCompatibleVersion("v1.1.0") && checkUpgradeBelowVer("v1.3.0") } } 404 steps { 405 runGinkgoVerifyManaged("multicluster/workloads/mccoherence", "${TEST_DUMP_ROOT}/mccoherence-workload", "false", "true", "true") 406 } 407 } 408 stage('deployment metrics') { 409 steps { 410 runGinkgoVerifyParallel('Prometheus pre-upgrade', 'metrics/deploymetrics', "${TEST_DUMP_ROOT}/prompreupgrade", "false", "true", "true", "deploymetrics") 411 } 412 } 413 } 414 } 415 stage('upgrade-platform-operator using kubectl') { 416 when { expression { return params.INSTALLATION_METHOD_UPGRADE == "kubectl" } } 417 stages { 418 stage("upgrade-platform-operator") { 419 steps { 420 upgradePlatformOperator() 421 } 422 } 423 stage("Verify Upgrade Required") { 424 steps { 425 runGinkgoRandomizeSerial('upgrade/pre-upgrade/verify-upgrade-required') 426 } 427 } 428 } 429 } 430 stage("upgrade-verrazzano") { 431 steps { 432 upgradeVerrazzano() 433 } 434 post { 435 always { 436 performDescribeVerrazzanoResource() 437 } 438 } 439 } 440 } 441 post { 442 failure { 443 script { 444 dumpK8sCluster("${WORKSPACE}/multicluster-install-cluster-snapshot") 445 } 446 } 447 } 448 } 449 450 stage('Verify Upgrade') { 451 // Rerun some stages to verify the upgrade 452 parallel { 453 stage ('verify-register') { 454 when { expression { isMultiCluster() } } 455 steps { 456 verifyRegisterManagedClusters(false) 457 } 458 } 459 stage ('verify-permissions') { 460 when { expression { isMultiCluster() } } 461 steps { 462 verifyManagedClusterPermissions() 463 } 464 } 465 stage ('system component metrics') { 466 steps { 467 runGinkgoRandomizeParallel('metrics/syscomponents') 468 } 469 } 470 } 471 post { 472 failure { 473 script { 474 dumpK8sCluster("${WORKSPACE}/multicluster-verify-upgrade-cluster-snapshot") 475 } 476 } 477 } 478 } 479 stage ('Verify Install') { 480 parallel { 481 stage('verify-install') { 482 steps { 483 runGinkgoRandomizeSerial('verify-install') 484 } 485 } 486 487 stage ('mc verify-install') { 488 when { expression { isMultiCluster() } } 489 steps { 490 runGinkgoRandomizeParallel('multicluster/verify-install') 491 } 492 } 493 } 494 } 495 496 // Run Verify Registry 497 stage('verify-registry') { 498 steps { 499 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 500 script { 501 runMulticlusterVerifyRegistry() 502 } 503 } 504 } 505 post { 506 always { 507 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 508 junit testResults: '**/*test-result.xml', allowEmptyResults: true 509 } 510 } 511 } 512 513 stage ('Verify Infra') { 514 parallel { 515 stage('verify-scripts') { 516 steps { 517 runGinkgoRandomizeParallel('scripts/install') 518 } 519 } 520 stage('verify-infra restapi') { 521 steps { 522 runGinkgoRandomizeParallel('verify-infra/restapi') 523 } 524 } 525 stage('verify-infra oam') { 526 steps { 527 runGinkgoRandomizeParallel('verify-infra/oam') 528 } 529 } 530 stage('verify-infra vmi') { 531 steps { 532 runGinkgoRandomizeParallel('verify-infra/vmi') 533 } 534 } 535 } 536 post { 537 always { 538 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 539 junit testResults: '**/*test-result.xml', allowEmptyResults: true 540 } 541 } 542 } 543 stage('Post-upgrade Acceptance Tests') { 544 stages { 545 stage ('Console') { 546 when { expression { isMultiCluster() } } 547 steps { 548 runConsoleTests() 549 } 550 post { 551 always { 552 saveConsoleTestArtifacts() 553 } 554 } 555 } 556 stage('Post-upgrade Acceptance Tests parallel') { 557 parallel { 558 stage('post-upgrade verify keycloak') { 559 steps { 560 runGinkgoRandomizeSerialAdmin('upgrade/post-upgrade/keycloak', "${TEST_DUMP_ROOT}/keycloak") 561 } 562 } 563 stage('opensearch post-upgrade') { 564 steps { 565 runGinkgoRandomizeSerialAdmin('upgrade/post-upgrade/opensearch', "${TEST_DUMP_ROOT}/opensearch") 566 } 567 } 568 stage('Grafana post-upgrade') { 569 steps { 570 runGinkgoRandomizeSerialAdmin('upgrade/post-upgrade/grafana', "${TEST_DUMP_ROOT}/grafana") 571 } 572 } 573 stage('Metricsbinding post-upgrade') { 574 steps { 575 runGinkgoRandomizeSerialAdmin('upgrade/post-upgrade/metricsbinding', "${TEST_DUMP_ROOT}/metricsbinding") 576 } 577 } 578 stage('deployment metrics') { 579 steps { 580 runGinkgoVerifyParallel('Prometheus post-upgrade', 'metrics/deploymetrics', "${TEST_DUMP_ROOT}/prompostupgrade", "true", "false", "false", "deploymetrics") 581 } 582 } 583 stage ("Verify install scripts") { 584 steps { 585 runGinkgoRandomizeAdmin("scripts/install", "${TEST_DUMP_ROOT}/install-scripts") 586 } 587 } 588 stage ('mc verify-api') { 589 when { expression { isMultiCluster() } } 590 steps { 591 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 592 script { 593 runMulticlusterVerifyApi() 594 } 595 } 596 } 597 post { 598 failure { 599 script { 600 dumpK8sCluster("${WORKSPACE}/multicluster-post-upgrade-acceptance-tests-cluster-snapshot-pre-uninstall") 601 } 602 } 603 } 604 } 605 } 606 } 607 stage ('Verify & Uninstall Example Apps') { 608 when { expression { isMultiCluster() } } 609 parallel { 610 stage ('mc examples helidon') { 611 steps { 612 runGinkgoVerifyManaged("multicluster/examples/helidon", "${TEST_DUMP_ROOT}/mchelidon-workload", "true", "false", "false") 613 } 614 } 615 stage ('mc coherence workload') { 616 steps { 617 runGinkgoVerifyManaged("multicluster/workloads/mccoherence", "${TEST_DUMP_ROOT}/mccoherence-workload", checkUpgradeBelowVer("v1.3.0"), "false", "false") 618 } 619 } 620 } 621 } 622 } 623 post { 624 failure { 625 script { 626 dumpK8sCluster("${WORKSPACE}/multicluster-post-upgrade-acceptance-tests-cluster-snapshot") 627 } 628 } 629 aborted { 630 script { 631 dumpK8sCluster("${WORKSPACE}/multicluster-post-upgrade-acceptance-tests-cluster-snapshot") 632 } 633 } 634 success { 635 script { 636 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 637 dumpK8sCluster("${WORKSPACE}/multicluster-post-upgrade-acceptance-tests-cluster-snapshot") 638 } 639 } 640 } 641 } 642 } 643 stage('Cleanup Tests') { 644 stages { 645 stage('verify deregister') { 646 when { expression { isMultiCluster() } } 647 steps { 648 verifyDeregisterManagedClusters() 649 } 650 } 651 } 652 post { 653 failure { 654 script { 655 dumpK8sCluster("${WORKSPACE}/multicluster-cleanup-tests-cluster-snapshot") 656 } 657 } 658 } 659 } 660 } 661 post { 662 failure { 663 pipelinePostFailure() 664 } 665 always { 666 script { 667 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 668 dumpAll() 669 } 670 } 671 copyGeneratedTestReports() 672 archiveArtifacts artifacts: "**/*-operator.yaml,**/install-verrazzano.yaml,**/kube_config,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*cluster-snapshot*/**,**/*bug-report*/**,**/full-cluster/**,**/${TEST_REPORT}", allowEmptyArchive: true 673 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 674 675 script { 676 failBuildIfFailuresDuringArtifactsDump() 677 } 678 } 679 cleanup { 680 deleteAll() 681 } 682 } 683 } 684 685 // false : version parameter falls between vz to(current) and from(to be upgraded) version 686 // true : vz to(current) and from(to be upgraded) version falls below version parameter 687 def checkUpgradeBelowVer(version) { 688 if(params.VERSION_FOR_UPGRADE != "NONE" && params.VERSION_FOR_UPGRADE != "current_branch"){ 689 VERRAZZANO_DEV_VERSION = VERSION_FOR_UPGRADE_ENV 690 } 691 return !(isMinVersion(params.VERSION_FOR_INSTALL, version) && !isMinVersion(VERRAZZANO_DEV_VERSION, version)) 692 } 693 694 // compares the two vz versions v1 & v2 695 // true : v1 < v2, false : v1 >= v2 696 def isMinVersion(v1, v2) { 697 v2 = v2 - 'v' 698 v1 = v1 - 'v' 699 def v2Split = v2.split('\\.') 700 runningVZVersion = v2Split[0] + '.' + v2Split[1] + '.' + v2Split[2].charAt(0) 701 v1Split = v1.split('\\.') 702 if(v1Split[0] < v2Split[0]){ 703 return true 704 }else if(v1Split[0] == v2Split[0] && v1Split[1] < v2Split[1]){ 705 return true 706 }else if(v1Split[0] == v2Split[0] && v1Split[1] == v2Split[1] && v1Split[2].charAt(0) < v2Split[2].charAt(0)){ 707 return true 708 } 709 return false 710 } 711 712 713 714 def createOCIDNSZone() { 715 script { 716 dns_zone_ocid = sh(script: "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId}", returnStdout: true) 717 } 718 } 719 720 def saveConsoleTestArtifacts() { 721 script { 722 sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh" 723 } 724 } 725 726 def createTestOCIConfigSecret() { 727 script { 728 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 729 for(int count=1; count<=clusterCount; count++) { 730 sh """ 731 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 732 cd ${GO_REPO_PATH}/verrazzano 733 ./tests/e2e/config/scripts/create-test-oci-config-secret.sh 734 """ 735 } 736 } 737 } 738 739 def deleteAll() { 740 // Delete clusters as the very first thing in cleanup to reclaim cluster resources especially OKE resources 741 deleteClusters() 742 deleteDir() 743 } 744 745 def deleteClusters() { 746 script { 747 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 748 if (env.TEST_ENV == "KIND") { 749 for(int count=1; count<=clusterCount; count++) { 750 sh """ 751 if [ "${env.TEST_ENV}" == "KIND" ] 752 then 753 kind delete cluster --name ${CLUSTER_NAME_PREFIX}-$count 754 fi 755 """ 756 } 757 } else { 758 deleteOkeClusters() 759 } 760 } 761 } 762 763 def copyGeneratedTestReports() { 764 script { 765 sh """ 766 # Copy the generated test reports to WORKSPACE to archive them 767 mkdir -p ${TEST_REPORT_DIR} 768 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 769 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 770 """ 771 } 772 } 773 774 def failBuildIfFailuresDuringArtifactsDump() { 775 script { 776 sh """ 777 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 778 echo "Failures seen during dumping of artifacts, treat post as failed" 779 exit 1 780 fi 781 """ 782 } 783 } 784 785 def printNodeLabels() { 786 script { 787 sh """ 788 echo "${NODE_LABELS}" 789 """ 790 } 791 } 792 793 def performNetRCOperations() { 794 script { 795 sh """ 796 cp -f "${NETRC_FILE}" $HOME/.netrc 797 chmod 600 $HOME/.netrc 798 """ 799 } 800 } 801 802 def performVerrazzanoDirectoryOperations() { 803 script { 804 sh """ 805 rm -rf ${GO_REPO_PATH}/verrazzano 806 mkdir -p ${GO_REPO_PATH}/verrazzano 807 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 808 """ 809 } 810 } 811 812 // Utility method to create a map that will be used later to perform parallel operations against multiple clusters 813 Map createClusterExecutionsMap() { 814 script { 815 // Create a dictionary of steps to be executed in parallel 816 // - the first one will always be the Admin cluster 817 // - clusters 2-max are managed clusters 818 def clusterExecutionsMap = [:] 819 return clusterExecutionsMap 820 } 821 } 822 823 // Create a KinD cluster instance 824 // - count - the cluster index into $KUBECONFIG_DIR 825 // - metallbAddressRange - the address range to provide the Metallb install within the KinD Docker bridge network address range 826 // - cleanupKindContainers - indicates to the script whether or not to remove any existing clusters with the same name before creating the new one 827 // - connectJenkinsRunnerToNetwork - indicates whether or not to connect the KinD Docker bridge network to the Jenkins local docker network 828 def installKindCluster(count, metallbAddressRange, cleanupKindContainers, connectJenkinsRunnerToNetwork) { 829 // For parallel execution, wrap this in a Groovy enclosure {} 830 return script { 831 sh """ 832 echo ${CLUSTER_NAME_PREFIX}-$count 833 echo ${KUBECONFIG_DIR}/$count/kube_config 834 mkdir -p ${KUBECONFIG_DIR}/$count 835 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 836 echo "Create Kind cluster \$1" 837 cd ${TEST_SCRIPTS_DIR} 838 # As a stop gap, for now we are using the api/vpo caches here to see if it helps with rate limiting issues, we will need to add specific caches so for now 839 # specify the cache name based on the count value, this is assuming 1 or 2 clusters 840 case "${count}" in 841 1) 842 ./create_kind_cluster.sh "${CLUSTER_NAME_PREFIX}-$count" "${GO_REPO_PATH}/verrazzano/platform-operator" "${KUBECONFIG_DIR}/$count/kube_config" "${params.KIND_CLUSTER_VERSION}" "$cleanupKindContainers" "$connectJenkinsRunnerToNetwork" true ${params.CREATE_CLUSTER_USE_CALICO} "vpo_integ" 843 ;; 844 2) 845 ./create_kind_cluster.sh "${CLUSTER_NAME_PREFIX}-$count" "${GO_REPO_PATH}/verrazzano/platform-operator" "${KUBECONFIG_DIR}/$count/kube_config" "${params.KIND_CLUSTER_VERSION}" "$cleanupKindContainers" "$connectJenkinsRunnerToNetwork" true ${params.CREATE_CLUSTER_USE_CALICO} "apo_integ" 846 ;; 847 *) 848 ./create_kind_cluster.sh "${CLUSTER_NAME_PREFIX}-$count" "${GO_REPO_PATH}/verrazzano/platform-operator" "${KUBECONFIG_DIR}/$count/kube_config" "${params.KIND_CLUSTER_VERSION}" "$cleanupKindContainers" "$connectJenkinsRunnerToNetwork" false ${params.CREATE_CLUSTER_USE_CALICO} "NONE" 849 ;; 850 esac 851 if [ ${params.CREATE_CLUSTER_USE_CALICO} == true ]; then 852 echo "Install Calico" 853 cd ${GO_REPO_PATH}/verrazzano 854 ./ci/scripts/install_calico.sh "${CLUSTER_NAME_PREFIX}-$count" 855 fi 856 kubectl wait --for=condition=ready nodes/${CLUSTER_NAME_PREFIX}-$count-control-plane --timeout=5m 857 kubectl wait --for=condition=ready pods/kube-controller-manager-${CLUSTER_NAME_PREFIX}-$count-control-plane -n kube-system --timeout=5m 858 echo "Listing pods in kube-system namespace ..." 859 kubectl get pods -n kube-system 860 echo "Install metallb" 861 cd ${GO_REPO_PATH}/verrazzano 862 ./tests/e2e/config/scripts/install-metallb.sh $metallbAddressRange 863 echo "Deploy external es and create its secret on the admin cluster if EXTERNAL_ELASTICSEARCH is true" 864 CLUSTER_NUMBER=${count} ${CREATE_EXTERNAL_OPENSEARCH} 865 """ 866 } 867 } 868 869 def deleteOkeClusters() { 870 script { 871 sh """ 872 mkdir -p ${KUBECONFIG_DIR} 873 if [ "${TEST_ENV}" == "ocidns_oke" ]; then 874 cd ${GO_REPO_PATH}/verrazzano 875 ./tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} || echo "Failed to delete DNS zone z${zoneId}" 876 fi 877 cd ${TEST_SCRIPTS_DIR} 878 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=multicluster-${env.BUILD_NUMBER}-${env.BRANCH_NAME} ./delete_oke_cluster.sh "$clusterCount" "${KUBECONFIG_DIR}" || true 879 """ 880 } 881 } 882 883 // Either download the specified release of the platform operator YAML, or create one 884 // using the specific operator image provided by the user. 885 def getVerrazzanoOperatorYaml() { 886 script { 887 OPERATOR_YAML_FILE=sh(returnStdout: true, script: "ci/scripts/derive_operator_yaml.sh ${params.VERSION_FOR_INSTALL}").trim() 888 RELEASE_NUMBER=sh(returnStdout: true, script: "echo ${VERSION_FOR_INSTALL} | tail -c4").trim() 889 890 sh """ 891 echo "Platform Operator Configuration" 892 cd ${GO_REPO_PATH}/verrazzano 893 if [ "NONE" == "${params.VERRAZZANO_OPERATOR_IMAGE}" ]; then 894 if [ "master" == $VERSION_FOR_UPGRADE_ENV ]; then 895 echo "Downloading operator from object storage" 896 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/release-${RELEASE_NUMBER}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-release-operator.yaml 897 else 898 echo "Downloading ${OPERATOR_YAML_FILE} for release ${params.VERSION_FOR_INSTALL}" 899 wget "${OPERATOR_YAML_FILE}" -O "${WORKSPACE}"/downloaded-release-operator.yaml 900 fi 901 cp ${WORKSPACE}/downloaded-release-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 902 else 903 echo "Generating operator.yaml based on image name provided: ${params.VERRAZZANO_OPERATOR_IMAGE}" 904 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${params.VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 905 fi 906 """ 907 } 908 } 909 910 // Install Verrazzano on each of the clusters 911 def installVerrazzano() { 912 script { 913 def verrazzanoInstallStages = createClusterExecutionsMap() 914 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 915 for(int count=1; count<=clusterCount; count++) { 916 def installerPath="${KUBECONFIG_DIR}/${count}/${installerFileName}" 917 def key = "vz-mgd-${count-1}" 918 if (count == 1) { 919 key = "vz-admin" 920 } 921 verrazzanoInstallStages["${key}"] = installVerrazzanoOnCluster(count, installerPath) 922 } 923 parallel verrazzanoInstallStages 924 } 925 } 926 927 // Install Verrazzano on a target cluster 928 // - count is the cluster index into the $KUBECONFIG_DIR 929 // - verrazzanoConfig is the Verrazzano CR to use to install VZ on the cluster 930 def installVerrazzanoOnCluster(count, verrazzanoConfig) { 931 // For parallel execution, wrap this in a Groovy enclosure {} 932 return { 933 script { 934 sh """ 935 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 936 cd ${GO_REPO_PATH}/verrazzano 937 938 # Display the kubectl and cluster versions 939 kubectl version 940 941 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/cluster${count}/pre-install-resources 942 943 if { [ 'false' == '${params.EXTERNAL_ELASTICSEARCH}' ]; } || { [ '${count}' != 1 ] && [ 'true' == '${params.EXTERNAL_ELASTICSEARCH}' ]; }; then 944 echo "Create the verrazzano-install namespace" 945 kubectl create namespace verrazzano-install 946 fi 947 948 echo "Create Image Pull Secrets" 949 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 950 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 951 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 952 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 953 954 echo "Following is the verrazzano CR file for the installation:" 955 cat ${verrazzanoConfig} 956 957 if [ "${params.INSTALLATION_METHOD_PREUPGRADE}" == "kubectl" ]; then 958 echo "Installing the Verrazzano Platform Operator" 959 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 960 961 # create secret in verrazzano-install ns 962 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 963 964 echo "Wait for Operator to be ready" 965 cd ${GO_REPO_PATH}/verrazzano 966 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 967 968 echo "Applying \$verrazzanoConfig" 969 # validate is set to false since we are using newer Verrazzano CR yaml when installing an older pre-upgrade version 970 kubectl apply --validate=false -f ${verrazzanoConfig} 971 972 # wait for Verrazzano install to complete 973 ./tests/e2e/config/scripts/wait-for-verrazzano-install.sh 974 elif [ "${params.INSTALLATION_METHOD_PREUPGRADE}" == "vz cli" ]; then 975 # Install Verrazzano 976 ${GO_REPO_PATH}/vz install --kubeconfig ${KUBECONFIG_DIR}/${count}/kube_config --filename ${verrazzanoConfig} --manifests ${WORKSPACE}/acceptance-test-operator.yaml 977 fi 978 """ 979 RUNNING_OPERATOR_VERSION = sh(returnStdout: true, script: "export KUBECONFIG='${KUBECONFIG_DIR}/${count}/kube_config' && kubectl get verrazzano -o jsonpath='{.items[0].status.version}'").trim() 980 echo "Verrazzano Version INSTALl : ${RUNNING_OPERATOR_VERSION}" 981 } 982 } 983 } 984 985 def getPlatformOperatorForRelease(){ 986 script { 987 //Removing v from the upgrade version for error handling. 988 VERSION_FOR_UPGRADE_ENV = VERSION_FOR_UPGRADE_ENV - "v" 989 //Command to retrieve the platform operator yaml of the current branch. 990 def operatorCMD = "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file" 991 992 if(params.VERSION_FOR_UPGRADE == "master"){ 993 operatorCMD = "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name master/operator.yaml --file" 994 995 //Command to retrieve the platform operator yaml of the release branch for patch release upgrade testing. 996 }else if(params.VERSION_FOR_UPGRADE != "current_branch" && params.IS_TRIGGERED_MANUALLY != "YES" && params.VERSION_FOR_UPGRADE != "NONE"){ 997 998 def latestReleaseBranch = getLatestReleaseVersion() 999 1000 operatorCMD = "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name release-${latestReleaseBranch}/operator.yaml --file" 1001 //Command to retrieve the released version platform operator yaml. 1002 }else if(params.IS_TRIGGERED_MANUALLY == "YES" && params.VERSION_FOR_UPGRADE != "NONE" && params.VERSION_FOR_UPGRADE != "current_branch"){ 1003 OPERATOR_YAML_FILE=sh(returnStdout: true, script: "ci/scripts/derive_operator_yaml.sh v${VERSION_FOR_UPGRADE_ENV}").trim() 1004 operatorCMD = "curl -L ${OPERATOR_YAML_FILE} --output" 1005 } 1006 1007 sh(script: "$operatorCMD $TARGET_UPGRADE_OPERATOR_YAML") 1008 } 1009 } 1010 1011 def getLatestReleaseVersion(){ 1012 def latestReleaseBranch 1013 if(params.VERSION_FOR_UPGRADE != "NONE" && params.VERSION_FOR_UPGRADE != "current_branch"){ 1014 VERSION_FOR_UPGRADE_ENV = VERSION_FOR_UPGRADE_ENV - "v" 1015 def latestReleaseTagSplit = VERSION_FOR_UPGRADE_ENV.split("\\.") 1016 latestReleaseBranch = latestReleaseTagSplit[0] + "." + latestReleaseTagSplit[1] 1017 } 1018 return latestReleaseBranch 1019 } 1020 1021 // Upgrade the verrazzano-platform-operator for all clusters 1022 def upgradePlatformOperator() { 1023 script { 1024 echo "Downloading target upgrade operator yaml" 1025 getPlatformOperatorForRelease() 1026 1027 def verrazzanoPlatformOperatorUpgradeStages= createClusterExecutionsMap() 1028 1029 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1030 for(int count = 1; count <= clusterCount; count++) { 1031 def key = "vz-mgd-${count-1}" 1032 if (count == 1) { 1033 key = "vz-admin" 1034 } 1035 verrazzanoPlatformOperatorUpgradeStages["${key}"] = upgradePlatformOperatorOnCluster(count) 1036 } 1037 parallel verrazzanoPlatformOperatorUpgradeStages 1038 } 1039 } 1040 1041 // Upgrade the verrazzano-platform-operator for a given cluster 1042 // - count is the cluster index into the $KUBECONFIG_DIR 1043 def upgradePlatformOperatorOnCluster(count) { 1044 // For parallel execution, wrap this in a Groovy enclosure {} 1045 return { 1046 script { 1047 def upgradeOperatorFile = "${KUBECONFIG_DIR}/${count}/upgrade-operator.yaml" 1048 def kubeClusterConfig="${KUBECONFIG_DIR}/${count}/kube_config" 1049 sh """ 1050 cp ${TARGET_UPGRADE_OPERATOR_YAML} ${upgradeOperatorFile} 1051 echo "Upgrading the Verrazzano platform operator" 1052 kubectl --kubeconfig=${kubeClusterConfig} apply -f ${upgradeOperatorFile} 1053 1054 # need to sleep since the old operator needs to transition to terminating state 1055 sleep 15 1056 1057 # ensure operator pod is up 1058 kubectl --kubeconfig=${kubeClusterConfig} -n verrazzano-install rollout status deployment/verrazzano-platform-operator 1059 kubectl --kubeconfig=${kubeClusterConfig} -n verrazzano-install rollout status deployment/verrazzano-platform-operator-webhook 1060 """ 1061 } 1062 } 1063 } 1064 1065 // Upgrade Verrazzano 1066 def upgradeVerrazzano() { 1067 script { 1068 // Download the bom for the target version that we are upgrading to, then extract the version 1069 // Note, this version will have a semver suffix which is generated for each build, e.g. 1.0.1-33+d592fed6 1070 if(params.VERSION_FOR_UPGRADE == "master"){ 1071 VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name master/generated-verrazzano-bom.json --file - | jq -r '.version'").trim() 1072 }else if(params.VERSION_FOR_UPGRADE != "NONE" && params.VERSION_FOR_UPGRADE != "current_branch"){ 1073 VERRAZZANO_DEV_VERSION = VERSION_FOR_UPGRADE_ENV 1074 }else{ 1075 VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file - | jq -r '.version'").trim() 1076 } 1077 1078 echo "Downloading target upgrade operator yaml" 1079 getPlatformOperatorForRelease() 1080 1081 def verrazzanoUpgradeStages = createClusterExecutionsMap() 1082 1083 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1084 for(int count=1; count<=clusterCount; count++) { 1085 def key = "vz-mgd-${count-1}" 1086 if (count == 1) { 1087 key = "vz-admin" 1088 } 1089 verrazzanoUpgradeStages["${key}"] = upgradeVerrazzanoOnCluster(count, "${VERRAZZANO_DEV_VERSION}") 1090 } 1091 parallel verrazzanoUpgradeStages 1092 } 1093 } 1094 1095 // Upgrade Verrazzano on a target cluster 1096 // - count is the cluster index into the $KUBECONFIG_DIR 1097 // - verrazzanoDevVersion is the Verrazzano version to be upgraded to 1098 def upgradeVerrazzanoOnCluster(count, verrazzanoDevVersion) { 1099 // For parallel execution, wrap this in a Groovy enclosure {} 1100 return { 1101 script { 1102 def kubeClusterConfig="${KUBECONFIG_DIR}/${count}/kube_config" 1103 def upgradeOperatorFile = "${KUBECONFIG_DIR}/${count}/upgrade-operator.yaml" 1104 1105 def v8oInstallFile="${KUBECONFIG_DIR}/${count}/${installerFileName}" 1106 def v8oUpgradeFile="${KUBECONFIG_DIR}/${count}/verrazzano-upgrade-cr.yaml" 1107 sh """ 1108 cp ${TARGET_UPGRADE_OPERATOR_YAML} ${upgradeOperatorFile} 1109 1110 mkdir -p ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/${count} 1111 1112 # Get the install job in verrazzano-install namespace 1113 kubectl --kubeconfig=${kubeClusterConfig} -n verrazzano-install get job -o yaml --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/${count}/verrazzano-pre-upgrade-job.out 1114 1115 if [ "${params.INSTALLATION_METHOD_UPGRADE}" == "kubectl" ]; then 1116 echo "Upgrading the Verrazzano installation to version" ${verrazzanoDevVersion} 1117 # Modify the version field in the Verrazzano CR file to be this new Verrazzano version 1118 cd ${GO_REPO_PATH}/verrazzano 1119 cp ${v8oInstallFile} ${v8oUpgradeFile} 1120 ${TEST_SCRIPTS_DIR}/process_upgrade_yaml.sh ${verrazzanoDevVersion} ${v8oUpgradeFile} 1121 1122 # Do the upgrade 1123 echo "Following is the verrazzano CR file with the new version:" 1124 cat ${v8oUpgradeFile} 1125 kubectl --kubeconfig=${kubeClusterConfig} patch verrazzano my-verrazzano -p '{"spec":{"version":"'${verrazzanoDevVersion}'"}}' --type=merge 1126 # wait for the upgrade to complete 1127 kubectl --kubeconfig=${kubeClusterConfig} wait --timeout=35m --for=condition=UpgradeComplete verrazzano/my-verrazzano 1128 elif [ "${params.INSTALLATION_METHOD_UPGRADE}" == "vz cli" ]; then 1129 echo "Upgrading the Verrazzano installation to version" ${verrazzanoDevVersion} 1130 ${GO_REPO_PATH}/vz upgrade --kubeconfig ${kubeClusterConfig} --version ${verrazzanoDevVersion} --manifests ${upgradeOperatorFile} --timeout 35m 1131 fi 1132 1133 # Get the install job(s) and mke sure the it matches pre-install. If there is more than 1 job or the job changed, then it won't match 1134 kubectl --kubeconfig=${kubeClusterConfig} -n verrazzano-install get job -o yaml --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/${count}/verrazzano-post-upgrade-job.out 1135 1136 echo "Ensuring that the install job(s) in verrazzzano-system are identical pre and post install" 1137 cmp -s ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/${count}/verrazzano-pre-upgrade-job.out ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/${count}/verrazzano-post-upgrade-job.out 1138 1139 # ideally we don't need to wait here 1140 sleep 15 1141 echo "helm list : releases across all namespaces, after upgrading Verrazzano installation ..." 1142 helm --kubeconfig=${kubeClusterConfig} list -A 1143 """ 1144 RUNNING_OPERATOR_VERSION = sh(returnStdout: true, script: "export KUBECONFIG='${KUBECONFIG_DIR}/${count}/kube_config' && kubectl get vz -o jsonpath='{.items[0].status.version}'").trim() 1145 echo "Verrazzano Version Upgrade : ${RUNNING_OPERATOR_VERSION}" 1146 } 1147 } 1148 } 1149 1150 // register all managed clusters 1151 def registerManagedClusters() { 1152 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1153 script { 1154 def verrazzanoRegisterManagedClusterStages = createClusterExecutionsMap() 1155 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1156 for(int count=2; count<=clusterCount; count++) { 1157 verrazzanoRegisterManagedClusterStages["${count} - Register Managed Cluster"] = registerManagedCluster(count) 1158 } 1159 // ADMIN_VZ_VERSION_AT_REGISTRATION is used by verify register test 1160 env.ADMIN_VZ_VERSION_AT_REGISTRATION = sh(returnStdout: true, 1161 script:"KUBECONFIG=${ADMIN_KUBECONFIG} kubectl get verrazzano -o jsonpath='{.items[0].status.version}'").trim() 1162 print "Admin VZ version at registration is ${env.ADMIN_VZ_VERSION_AT_REGISTRATION}" 1163 parallel verrazzanoRegisterManagedClusterStages 1164 } 1165 } 1166 } 1167 1168 def registerManagedCluster(count) { 1169 // For parallel execution, wrap this in a Groovy enclosure {} 1170 return { 1171 script { 1172 sh """ 1173 export MANAGED_CLUSTER_DIR="${KUBECONFIG_DIR}/${count}" 1174 export MANAGED_CLUSTER_NAME="managed${count-1}" 1175 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1176 export MANAGED_CLUSTER_ENV="mgd${count-1}" 1177 cd ${GO_REPO_PATH}/verrazzano 1178 ./tests/e2e/config/scripts/register_managed_cluster.sh 1179 """ 1180 } 1181 } 1182 } 1183 1184 // Verify the register of the managed clusters 1185 def verifyRegisterManagedClusters(minimalVerification) { 1186 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1187 script { 1188 def verrazzanoRegisterManagedClusterStages = createClusterExecutionsMap() 1189 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1190 for(int count=2; count<=clusterCount; count++) { 1191 verrazzanoRegisterManagedClusterStages["${count} - Verify Register Managed Cluster"] = verifyRegisterManagedCluster(count, minimalVerification) 1192 } 1193 parallel verrazzanoRegisterManagedClusterStages 1194 } 1195 } 1196 } 1197 1198 def verifyRegisterManagedCluster(count, minimalVerification) { 1199 // For parallel execution, wrap this in a Groovy enclosure {} 1200 return { 1201 script { 1202 sh """ 1203 export MANAGED_CLUSTER_NAME="managed${count-1}" 1204 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1205 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1206 ginkgo build multicluster/verify-register/ 1207 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-register/*.test -- --minimalVerification=${minimalVerification} 1208 """ 1209 } 1210 } 1211 } 1212 1213 // Verify the deregister of the managed clusters 1214 def verifyDeregisterManagedClusters() { 1215 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1216 script { 1217 def verrazzanoDeregisterManagedClusterStages = createClusterExecutionsMap() 1218 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1219 for(int count=2; count<=clusterCount; count++) { 1220 verrazzanoDeregisterManagedClusterStages["${count} - Verify Deregister Managed Cluster"] = verifyDeregisterManagedCluster(count) 1221 } 1222 parallel verrazzanoDeregisterManagedClusterStages 1223 } 1224 } 1225 } 1226 1227 def verifyDeregisterManagedCluster(count) { 1228 // For parallel execution, wrap this in a Groovy enclosure {} 1229 return { 1230 script { 1231 sh """ 1232 export MANAGED_CLUSTER_NAME="managed${count-1}" 1233 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1234 export ADMIN_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1235 ./tests/e2e/config/scripts/deregister_managed_cluster.sh 1236 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1237 ginkgo build multicluster/verify-deregister/ 1238 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-deregister/*.test 1239 """ 1240 } 1241 } 1242 } 1243 1244 // Verify the managed cluster permissions 1245 def verifyManagedClusterPermissions() { 1246 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1247 script { 1248 def verrazzanoManagedClusterPermissionsStages = createClusterExecutionsMap() 1249 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1250 for(int count=2; count<=clusterCount; count++) { 1251 verrazzanoManagedClusterPermissionsStages["${count} - Verify Managed Cluster Permissions"] = verifyManagedClusterPermissions(count) 1252 } 1253 parallel verrazzanoManagedClusterPermissionsStages 1254 } 1255 } 1256 } 1257 1258 def verifyManagedClusterPermissions(count) { 1259 // For parallel execution, wrap this in a Groovy enclosure {} 1260 return { 1261 script { 1262 sh """ 1263 export MANAGED_CLUSTER_NAME="managed${count-1}" 1264 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1265 export MANAGED_ACCESS_KUBECONFIG="${KUBECONFIG_DIR}/${count}/managed_kube_config" 1266 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1267 ginkgo -v -stream --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-permissions/... 1268 """ 1269 } 1270 } 1271 } 1272 1273 // Run ginkgo test suites 1274 def runGinkgoVerifyManaged(testSuitePath, clusterDumpDirectory, skipDeploy, skipUndeploy, skipVerify) { 1275 script { 1276 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1277 sh """ 1278 export MANAGED_CLUSTER_NAME="managed1" 1279 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/2/kube_config" 1280 export CLUSTER_COUNT=$clusterCount 1281 export ADMIN_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1282 export CLUSTER_NAME="managed1" 1283 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1284 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/2/kube_config" 1285 export DUMP_DIRECTORY="${clusterDumpDirectory}" 1286 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --skipVerify=${skipVerify} 1287 """ 1288 } 1289 } 1290 1291 // Run a test suite against all clusters in serial 1292 def runGinkgoRandomizeSerial(testSuitePath) { 1293 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1294 script { 1295 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1296 def clusterName = "" 1297 for(int count=1; count<=clusterCount; count++) { 1298 // The first cluster created by this script is named as admin, and the subsequent clusters are named as 1299 // managed1, managed2, etc. 1300 if (count == 1) { 1301 clusterName="admin" 1302 } else { 1303 clusterName="managed${count-1}" 1304 } 1305 sh """ 1306 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1307 export CLUSTER_NAME="${clusterName}" 1308 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1309 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 1310 """ 1311 } 1312 } 1313 } 1314 } 1315 1316 // Run a test suite against all clusters in parallel 1317 def runGinkgoRandomizeParallel(testSuitePath) { 1318 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1319 script { 1320 def runGinkgoRandomizeStages = createClusterExecutionsMap() 1321 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1322 def clusterName = "" 1323 for(int count=1; count<=clusterCount; count++) { 1324 // The first cluster created by this script is named as admin, and the subsequent clusters are named as 1325 // managed1, managed2, etc. 1326 if (count == 1) { 1327 clusterName="admin" 1328 } else { 1329 clusterName="managed${count-1}" 1330 } 1331 sh """ 1332 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1333 export CLUSTER_NAME="${clusterName}" 1334 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1335 ginkgo build ${testSuitePath}/ 1336 """ 1337 runGinkgoRandomizeStages["${count} - Executing ginkgo test suite ${testSuitePath}"] = runGinkgoRandomize(count, clusterName, testSuitePath) 1338 } 1339 parallel runGinkgoRandomizeStages 1340 } 1341 } 1342 } 1343 1344 def runGinkgoRandomize(count, clusterName, testSuitePath) { 1345 // For parallel execution, wrap this in a Groovy enclosure {} 1346 return { 1347 script { 1348 sh """ 1349 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1350 export CLUSTER_NAME="${clusterName}" 1351 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1352 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/*.test 1353 """ 1354 } 1355 } 1356 } 1357 1358 // Run a test suite against just the admin cluster 1359 def runGinkgoRandomizeAdmin(testSuitePath, clusterDumpDirectory) { 1360 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1361 sh """ 1362 export KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1363 export CLUSTER_NAME="admin" 1364 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1365 export DUMP_DIRECTORY="${clusterDumpDirectory}" 1366 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1367 ginkgo build ${testSuitePath}/ 1368 ginkgo -p --randomize-all -v --keep-going --no-color -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/*.test 1369 """ 1370 } 1371 } 1372 1373 // Run a test suite serially against just the admin cluster 1374 def runGinkgoRandomizeSerialAdmin(testSuitePath, clusterDumpDirectory) { 1375 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1376 sh """ 1377 export KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1378 export CLUSTER_NAME="admin" 1379 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1380 export DUMP_DIRECTORY="${clusterDumpDirectory}" 1381 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1382 ginkgo build ${testSuitePath}/ 1383 ginkgo --randomize-all -v --keep-going --no-color -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/*.test 1384 """ 1385 } 1386 } 1387 1388 def runGinkgoVerify(testSuitePath, clusterDumpDirectory, skipDeploy, skipUndeploy, skipVerify, namespace) { 1389 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1390 sh """ 1391 export KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1392 export CLUSTER_NAME="admin" 1393 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1394 export DUMP_DIRECTORY="${clusterDumpDirectory}" 1395 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1396 ginkgo build ${testSuitePath}/ 1397 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/*.test -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --skipVerify=${skipVerify} --namespace=${namespace} 1398 """ 1399 } 1400 } 1401 1402 def runGinkgoVerifyParallel(testSuitePrefix, testSuitePath, clusterDumpDirectory, skipDeploy, skipUndeploy, skipVerify, namespace) { 1403 script { 1404 def runGinkgoVerifyStages = createClusterExecutionsMap() 1405 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1406 def clusterName = "" 1407 for(int count=1; count<=clusterCount; count++) { 1408 // The first cluster created by this script is named as admin, and the subsequent clusters are named as 1409 // managed1, managed2, etc. 1410 if (count == 1) { 1411 clusterName="admin" 1412 } else { 1413 clusterName="managed${count-1}" 1414 } 1415 runGinkgoVerifyStages["${testSuitePrefix} - ${clusterName}"] = runGinkgoVerifyInCluster(count, clusterName, testSuitePath, clusterDumpDirectory, skipDeploy, skipUndeploy, skipVerify, namespace) 1416 } 1417 parallel runGinkgoVerifyStages 1418 } 1419 } 1420 1421 def runGinkgoVerifyInCluster(count, clusterName, testSuitePath, clusterDumpDirectory, skipDeploy, skipUndeploy, skipVerify, namespace) { 1422 // For parallel execution, wrap this in a Groovy enclosure {} 1423 return { 1424 script { 1425 sh """ 1426 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1427 export CLUSTER_NAME=${clusterName} 1428 export ADMIN_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1429 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1430 export DUMP_DIRECTORY="${clusterDumpDirectory}${clusterName}" 1431 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1432 ginkgo build ${testSuitePath}/ 1433 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/*.test -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --skipVerify=${skipVerify} --namespace=${namespace} 1434 """ 1435 } 1436 } 1437 } 1438 1439 // Configure the Admin and Managed cluster installer custom resources 1440 def configureVerrazzanoInstallers(installResourceTemplate, configProcessorScript, String... extraArgs) { 1441 script { 1442 // Concatenate the variable args into a single string 1443 String allArgs = "" 1444 extraArgs.each { allArgs += it + " " } 1445 1446 def verrazzanoInstallerStages = createClusterExecutionsMap() 1447 1448 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1449 for(int count=1; count<=clusterCount; count++) { 1450 def destinationPath = "${env.KUBECONFIG_DIR}/${count}/${installerFileName}" 1451 def installProfile = MANAGED_CLUSTER_PROFILE 1452 // Installs using OCI DNS require a unique domain per cluster 1453 // - also, the env name must be <= 10 chars for some reason. 1454 def envName = "mgd${count-1}" 1455 if (count == 1) { 1456 // Cluster "1" is always the admin cluster, use the chosen profile for the VZ install 1457 // with the env name "admin" 1458 installProfile = ADMIN_CLUSTER_PROFILE 1459 envName = "admin" 1460 } 1461 verrazzanoInstallerStages["${envName}"] = configureVerrazzanoInstallerOnCluster(count, 1462 installResourceTemplate, destinationPath, envName, installProfile, configProcessorScript, allArgs) 1463 } 1464 parallel verrazzanoInstallerStages 1465 } 1466 } 1467 1468 def configureVerrazzanoInstallerOnCluster(count, installResourceTemplate, destinationPath, envName, installProfile, 1469 configProcessorScript, allArgs) { 1470 // For parallel execution, wrap this in a Groovy enclosure {} 1471 return { 1472 script { 1473 sh """ 1474 mkdir -p "${KUBECONFIG_DIR}/${count}" 1475 export PATH=${HOME}/go/bin:${PATH} 1476 cd ${GO_REPO_PATH}/verrazzano 1477 # Copy the template config over for the mgd cluster profile configuration 1478 cp $installResourceTemplate $destinationPath 1479 VZ_ENVIRONMENT_NAME="${envName}" INSTALL_PROFILE=$installProfile $configProcessorScript $destinationPath $allArgs 1480 """ 1481 } 1482 } 1483 } 1484 1485 // Create the required KinD clusters 1486 def createKindClusters() { 1487 script { 1488 sh """ 1489 echo "tests will execute" > ${TESTS_EXECUTED_FILE} 1490 """ 1491 // NOTE: Eventually we should be able to parallelize the cluster creation, however 1492 // we seem to be getting some kind of timing issue on cluster create; the 2nd 1493 // cluster always seems to get a connect/timeout issue, so for now we are keeping 1494 // the KinD cluster creation serial 1495 1496 // Can these for now, but eventually we should be able to build this based on 1497 // inspecting the Docker bridge network CIDR and splitting up a range based on 1498 // the cluster count. 1499 1500 def addressRanges = [ "172.18.0.231-172.18.0.238", "172.18.0.239-172.18.0.246", "172.18.0.247-172.18.0.254"] 1501 def clusterInstallStages = [:] 1502 boolean cleanupKindContainers = true 1503 boolean connectJenkinsRunnerToNetwork = true 1504 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1505 for(int count=1; count<=clusterCount; count++) { 1506 string metallbAddressRange = addressRanges.get(count-1) 1507 def deployStep = "cluster-${count}" 1508 // Create dictionary of steps for parallel execution 1509 //clusterInstallStages[deployStep] = installKindCluster(count, metallbAddressRange, cleanupKindContainers, connectJenkinsRunnerToNetwork) 1510 // For sequential execution of steps 1511 installKindCluster(count, metallbAddressRange, cleanupKindContainers, connectJenkinsRunnerToNetwork) 1512 cleanupKindContainers = false 1513 connectJenkinsRunnerToNetwork = false 1514 } 1515 // Execute steps in parallel 1516 //parallel clusterInstallStages 1517 } 1518 } 1519 1520 // Invoke the OKE cluster creation script for the desired number of clusters 1521 def createOKEClusters(clusterPrefix) { 1522 script { 1523 sh """ 1524 echo "tests will execute" > ${TESTS_EXECUTED_FILE} 1525 """ 1526 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1527 sh """ 1528 mkdir -p ${KUBECONFIG_DIR} 1529 echo "Create OKE cluster" 1530 cd ${TEST_SCRIPTS_DIR} 1531 TF_VAR_label_prefix=${clusterPrefix} TF_VAR_state_name=multicluster-${env.BUILD_NUMBER}-${env.BRANCH_NAME} ./create_oke_multi_cluster.sh "$clusterCount" "${KUBECONFIG_DIR}" ${params.CREATE_CLUSTER_USE_CALICO} 1532 """ 1533 } 1534 } 1535 1536 def dumpAll() { 1537 dumpVerrazzanoSystemPods() 1538 dumpCattleSystemPods() 1539 dumpNginxIngressControllerLogs() 1540 dumpVerrazzanoPlatformOperatorLogs() 1541 dumpVerrazzanoApplicationOperatorLogs() 1542 dumpOamKubernetesRuntimeLogs() 1543 dumpVerrazzanoApiLogs() 1544 } 1545 1546 1547 def dumpK8sCluster(dumpDirectory) { 1548 script { 1549 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 1550 def verrazzanoDumpK8sClusterStages = createClusterExecutionsMap() 1551 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1552 for (int count = 1; count <= clusterCount; count++) { 1553 verrazzanoDumpK8sClusterStages["${count} - Dump K8s Cluster"] = dumpK8sSpecificCluster(count, dumpDirectory) 1554 } 1555 parallel verrazzanoDumpK8sClusterStages 1556 } 1557 } 1558 } 1559 1560 def dumpK8sSpecificCluster(count, dumpDirectory) { 1561 // For parallel execution, wrap this in a Groovy enclosure {} 1562 return { 1563 script { 1564 sh """ 1565 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1566 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}/cluster-snapshot-${count} 1567 """ 1568 } 1569 } 1570 } 1571 1572 def dumpVerrazzanoSystemPods() { 1573 script { 1574 def dumpVerrazzanoSystemPodsStages = createClusterExecutionsMap() 1575 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1576 for(int count=1; count<=clusterCount; count++) { 1577 dumpVerrazzanoSystemPodsStages["${count} - Dump Verrazzano System Pods"] = dumpVerrazzanoSystemPods(count) 1578 } 1579 parallel dumpVerrazzanoSystemPodsStages 1580 } 1581 } 1582 1583 def dumpVerrazzanoSystemPods(count) { 1584 // For parallel execution, wrap this in a Groovy enclosure {} 1585 return { 1586 script { 1587 sh """ 1588 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1589 export LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1590 mkdir -p ${LOG_DIR} 1591 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-pods.log" 1592 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 1593 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-certs.log" 1594 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 1595 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-osd.log" 1596 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 1597 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-es-master.log" 1598 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 1599 """ 1600 } 1601 } 1602 } 1603 1604 def dumpCattleSystemPods() { 1605 script { 1606 def dumpCattleSystemPodsStages = createClusterExecutionsMap() 1607 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1608 for(int count=1; count<=clusterCount; count++) { 1609 dumpCattleSystemPodsStages["${count} - Dump Cattle System Pods"] = dumpCattleSystemPods(count) 1610 } 1611 parallel dumpCattleSystemPodsStages 1612 } 1613 } 1614 1615 def dumpCattleSystemPods(count) { 1616 // For parallel execution, wrap this in a Groovy enclosure {} 1617 return { 1618 script { 1619 sh """ 1620 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1621 export LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1622 mkdir -p ${LOG_DIR} 1623 export DIAGNOSTIC_LOG="${LOG_DIR}/cattle-system-pods.log" 1624 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 1625 export DIAGNOSTIC_LOG="${LOG_DIR}/rancher.log" 1626 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 1627 """ 1628 } 1629 } 1630 } 1631 1632 def dumpNginxIngressControllerLogs() { 1633 script { 1634 def dumpNginxIngressControllerLogsStages = createClusterExecutionsMap() 1635 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1636 for(int count=1; count<=clusterCount; count++) { 1637 dumpNginxIngressControllerLogsStages["${count} - Dump Nginx Ingress Controller Logs"] = dumpNginxIngressControllerLogs(count) 1638 } 1639 parallel dumpNginxIngressControllerLogsStages 1640 } 1641 } 1642 1643 def dumpNginxIngressControllerLogs(count) { 1644 // For parallel execution, wrap this in a Groovy enclosure {} 1645 return { 1646 script { 1647 sh """ 1648 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1649 export LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1650 mkdir -p ${LOG_DIR} 1651 export DIAGNOSTIC_LOG="${LOG_DIR}/nginx-ingress-controller.log" 1652 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 1653 """ 1654 } 1655 } 1656 } 1657 1658 def dumpVerrazzanoPlatformOperatorLogs() { 1659 script { 1660 def dumpVerrazzanoPlatformOperatorLogsStages = createClusterExecutionsMap() 1661 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1662 for(int count=1; count<=clusterCount; count++) { 1663 dumpVerrazzanoPlatformOperatorLogsStages["${count} - Dump Verrazzano Platform Operator Logs"] = dumpVerrazzanoPlatformOperatorLogs(count) 1664 } 1665 parallel dumpVerrazzanoPlatformOperatorLogsStages 1666 } 1667 } 1668 1669 def dumpVerrazzanoPlatformOperatorLogs(count) { 1670 // For parallel execution, wrap this in a Groovy enclosure {} 1671 return { 1672 script { 1673 sh """ 1674 ## dump out verrazzano-platform-operator logs 1675 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1676 export LOG_DIR="${WORKSPACE}/verrazzano-platform-operator/logs/cluster-$count" 1677 mkdir -p ${LOG_DIR} 1678 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${LOG_DIR}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 1679 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${LOG_DIR}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 1680 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 1681 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 1682 echo "------------------------------------------" 1683 """ 1684 } 1685 } 1686 } 1687 1688 def dumpVerrazzanoApplicationOperatorLogs() { 1689 script { 1690 def dumpVerrazzanoApplicationOperatorLogsStages = createClusterExecutionsMap() 1691 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1692 for(int count=1; count<=clusterCount; count++) { 1693 dumpVerrazzanoApplicationOperatorLogsStages["${count} - Dump Verrazzano Application Operator Logs"] = dumpVerrazzanoApplicationOperatorLogs(count) 1694 } 1695 parallel dumpVerrazzanoApplicationOperatorLogsStages 1696 } 1697 } 1698 1699 def dumpVerrazzanoApplicationOperatorLogs(count) { 1700 // For parallel execution, wrap this in a Groovy enclosure {} 1701 return { 1702 script { 1703 sh """ 1704 ## dump out verrazzano-application-operator logs 1705 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1706 export LOG_DIR="${WORKSPACE}/verrazzano-application-operator/logs/cluster-$count" 1707 mkdir -p ${LOG_DIR} 1708 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${LOG_DIR}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 1709 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${LOG_DIR}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 1710 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 1711 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 1712 echo "------------------------------------------" 1713 """ 1714 } 1715 } 1716 } 1717 1718 def dumpOamKubernetesRuntimeLogs() { 1719 script { 1720 def dumpOamKubernetesRuntimeLogsStages = createClusterExecutionsMap() 1721 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1722 for(int count=1; count<=clusterCount; count++) { 1723 dumpOamKubernetesRuntimeLogsStages["${count} - Dump Oam Kubernetes Runtime Logs"] = dumpOamKubernetesRuntimeLogs(count) 1724 } 1725 parallel dumpOamKubernetesRuntimeLogsStages 1726 } 1727 } 1728 1729 def dumpOamKubernetesRuntimeLogs(count) { 1730 // For parallel execution, wrap this in a Groovy enclosure {} 1731 return { 1732 script { 1733 sh """ 1734 ## dump out oam-kubernetes-runtime logs 1735 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1736 export LOG_DIR="${WORKSPACE}/oam-kubernetes-runtime/logs/cluster-$count" 1737 mkdir -p ${LOG_DIR} 1738 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${LOG_DIR}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 1739 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${LOG_DIR}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 1740 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 1741 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 1742 echo "------------------------------------------" 1743 """ 1744 } 1745 } 1746 } 1747 1748 def dumpVerrazzanoApiLogs() { 1749 script { 1750 def dumpVerrazzanoApiLogsStages = createClusterExecutionsMap() 1751 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1752 for(int count=1; count<=clusterCount; count++) { 1753 dumpVerrazzanoApiLogsStages["${count} - Dump Verrazzano Api Logs"] = dumpVerrazzanoApiLogs(count) 1754 } 1755 parallel dumpVerrazzanoApiLogsStages 1756 } 1757 } 1758 1759 def dumpVerrazzanoApiLogs(count) { 1760 // For parallel execution, wrap this in a Groovy enclosure {} 1761 return { 1762 script { 1763 sh """ 1764 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1765 export LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1766 mkdir -p ${LOG_DIR} 1767 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-authproxy.log" 1768 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 1769 """ 1770 } 1771 } 1772 } 1773 1774 def dumpInstallLogs() { 1775 script { 1776 def verrazzanoDumpInstallLogStages = createClusterExecutionsMap() 1777 1778 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1779 for(int count=1; count<=clusterCount; count++) { 1780 LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1781 1782 // This function may run on older versions of Verrazzano that have the logs stored in the default namespace 1783 def namespace = "verrazzano-install" 1784 if (params.VERSION_FOR_INSTALL.startsWith("v1.0.0") || params.VERSION_FOR_INSTALL.startsWith("v1.0.1")) { 1785 namespace = "default" 1786 } 1787 verrazzanoDumpInstallLogStages["${count} - Dump Install Logs"] = dumpInstallLogsOnCluster(count, LOG_DIR, namespace) 1788 } 1789 parallel verrazzanoDumpInstallLogStages 1790 } 1791 } 1792 1793 def dumpInstallLogsOnCluster(count, logDir, namespace) { 1794 // For parallel execution, wrap this in a Groovy enclosure {} 1795 return { 1796 script { 1797 sh """ 1798 ## dump Verrazzano install logs 1799 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1800 mkdir -p $logDir 1801 kubectl -n $namespace logs --selector=job-name=verrazzano-install-my-verrazzano > $logDir/${VERRAZZANO_INSTALL_LOG} --tail -1 1802 kubectl -n $namespace describe pod --selector=job-name=verrazzano-install-my-verrazzano > $logDir/verrazzano-install-job-pod.out 1803 kubectl get features > $logDir/verrazzano-install-features.out || true 1804 echo "------------------------------------------" 1805 """ 1806 } 1807 } 1808 } 1809 1810 def getEffectiveDumpOnSuccess() { 1811 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 1812 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 1813 effectiveValue = true 1814 echo "Forcing dump on success based on global override setting" 1815 } 1816 return effectiveValue 1817 } 1818 1819 def setDisplayName() { 1820 echo "Start setDisplayName" 1821 def causes = currentBuild.getBuildCauses() 1822 echo "causes: " + causes.toString() 1823 for (cause in causes) { 1824 def causeString = cause.toString() 1825 echo "current cause: " + causeString 1826 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 1827 echo "This job was caused by " + causeString 1828 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 1829 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 1830 } else if (causeString.contains("verrazzano-flaky-tests")) { 1831 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 1832 } 1833 } 1834 } 1835 echo "End setDisplayName" 1836 } 1837 1838 def modifyHelloHelidonApp(newNamespace, newProjName) { 1839 sh """ 1840 # create modified versions of the hello helidon MC example 1841 export MC_HH_DEST_DIR=${GO_REPO_PATH}/verrazzano/examples/multicluster/${newNamespace} 1842 export MC_HH_SOURCE_DIR=${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon 1843 export MC_APP_NAMESPACE="${newNamespace}" 1844 export MC_PROJ_NAME="${newProjName}" 1845 ${GO_REPO_PATH}/verrazzano/ci/scripts/generate_mc_hello_deployment_files.sh 1846 """ 1847 } 1848 1849 def runMulticlusterVerifyApi() { 1850 def verrazzanoMulticlusterVerifyApiStages = createClusterExecutionsMap() 1851 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1852 for(int count=2; count<=clusterCount; count++) { 1853 verrazzanoMulticlusterVerifyApiStages["${count} - Verify Api"] = runMulticlusterVerifyApi(count) 1854 } 1855 parallel verrazzanoMulticlusterVerifyApiStages 1856 } 1857 1858 def runMulticlusterVerifyApi(count) { 1859 // For parallel execution, wrap this in a Groovy enclosure {} 1860 return { 1861 script { 1862 sh """ 1863 export MANAGED_CLUSTER_NAME="managed${count-1}" 1864 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1865 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1866 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-api/... 1867 """ 1868 } 1869 } 1870 } 1871 1872 def runMulticlusterVerifyRegistry() { 1873 def verifyRegistryStages = createClusterExecutionsMap() 1874 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1875 for(int count=1; count<=clusterCount; count++) { 1876 verifyRegistryStages["${count} - Verify registry"] = verifyRegistry(count) 1877 } 1878 sh """ 1879 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1880 ginkgo build registry/ 1881 """ 1882 parallel verifyRegistryStages 1883 } 1884 1885 def verifyRegistry(count) { 1886 // For parallel execution, wrap this in a Groovy enclosure {} 1887 return { 1888 script { 1889 sh """ 1890 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1891 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1892 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" registry/*.test 1893 """ 1894 } 1895 } 1896 } 1897 1898 def runConsoleTests() { 1899 // Set app information used by the application page UI tests to assert for app info 1900 // Console runs on admin cluster and the KUBECONFIG is pointed at it (which is cluster 1) 1901 // Make sure that application page tests are also run by setting RUN_APP_TESTS=true since we deployed 1902 // a sample app for that purpose 1903 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 1904 sh """ 1905 export DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console" 1906 export CONSOLE_REPO_BRANCH="${params.CONSOLE_REPO_BRANCH}" 1907 export CONSOLE_APP_NAME="${SAMPLE_APP_NAME}" 1908 export CONSOLE_APP_NAMESPACE="${SAMPLE_APP_NAMESPACE}" 1909 export CONSOLE_APP_CLUSTER="managed1" 1910 export CONSOLE_APP_COMP="${SAMPLE_APP_COMPONENT}" 1911 KUBECONFIG=${ADMIN_KUBECONFIG} RUN_APP_TESTS=true ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh 1912 """ 1913 } 1914 } 1915 1916 def setVZConfigForInstallation(){ 1917 if(params.CRD_API_VERSION == "v1beta1"){ 1918 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-vz-prod-kind-upgrade.yaml" 1919 INSTALL_CONFIG_FILE_OCIDNS = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-verrazzano-ocidns.yaml" 1920 INSTALL_CONFIG_FILE_NIPIO = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-verrazzano-nipio.yaml" 1921 } 1922 if(params.VZ_INSTALL_CONFIG == "dev-kind-persistence" && params.CRD_API_VERSION == "v1beta1"){ 1923 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-verrazzano-kind-with-persistence.yaml" 1924 ADMIN_CLUSTER_PROFILE = "dev" 1925 MANAGED_CLUSTER_PROFILE = "managed-cluster" 1926 }else if(params.VZ_INSTALL_CONFIG == "dev-kind-persistence" && params.CRD_API_VERSION == "v1alpha1") { 1927 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-kind-with-persistence.yaml" 1928 ADMIN_CLUSTER_PROFILE = "dev" 1929 MANAGED_CLUSTER_PROFILE = "managed-cluster" 1930 } 1931 } 1932 1933 def isMultiCluster() { 1934 return params.TOTAL_CLUSTERS.toInteger() >= 2 1935 } 1936 1937 def isVZOperatorMinimumCompatibleVersion(minimumOperatorVersion){ 1938 1939 vzOperatorVersion = RUNNING_OPERATOR_VERSION - 'v' 1940 minimumOperatorVersion = minimumOperatorVersion - 'v' 1941 def vzOperatorVersionSplit = vzOperatorVersion.split('\\.') 1942 runningVZVersion = vzOperatorVersionSplit[0] + '.' + vzOperatorVersionSplit[1] + '.' + vzOperatorVersionSplit[2].charAt(0) 1943 echo "Verrazzano Version is: ${runningVZVersion}" 1944 //Splitting the string minimumOperatorVersion to compare minor and patch versions. 1945 minimumOperatorVersionSplit = minimumOperatorVersion.split('\\.') 1946 1947 //Returns true -> when the vzOperatorVersion minor version is greater than the minor version of user defined minimumOperatorVersion 1948 if(vzOperatorVersionSplit[0] > minimumOperatorVersionSplit[0]){ 1949 return true 1950 //Returns true -> vzOperatorVersion minor version is greater/equal than the user defined minimumOperatorVersion 1951 }else if(vzOperatorVersionSplit[0] == minimumOperatorVersionSplit[0] && minimumOperatorVersionSplit[1] < vzOperatorVersionSplit[1]){ 1952 echo "${vzOperatorVersionSplit[0]} >= ${minimumOperatorVersionSplit[0]} && ${minimumOperatorVersionSplit[1]} < ${vzOperatorVersionSplit[1]}" 1953 return true 1954 // Returns true -> when vzOperatorVersion and minimumOperatorVersion has equal minor versions but vzOperatorVersion patch is greater/equal to the user specified minimumOperatorVersion 1955 }else if(vzOperatorVersionSplit[0] == minimumOperatorVersionSplit[0] && minimumOperatorVersionSplit[1] == vzOperatorVersionSplit[1] && vzOperatorVersionSplit[2].charAt(0) >= minimumOperatorVersionSplit[2].charAt(0)){ 1956 echo "${vzOperatorVersionSplit[0]} == ${minimumOperatorVersionSplit[0]} && ${minimumOperatorVersionSplit[1]} == ${vzOperatorVersionSplit[1]} && ${vzOperatorVersionSplit[2]} >= ${minimumOperatorVersionSplit[2]}" 1957 return true 1958 } 1959 echo "${vzOperatorVersionSplit[0]} > ${minimumOperatorVersionSplit[0]}" 1960 echo "${vzOperatorVersionSplit[0]} >= ${minimumOperatorVersionSplit[0]} && ${minimumOperatorVersionSplit[1]} < ${vzOperatorVersionSplit[1]}" 1961 echo "${vzOperatorVersionSplit[0]} == ${minimumOperatorVersionSplit[0]} && ${minimumOperatorVersionSplit[1]} == ${vzOperatorVersionSplit[1]} && ${vzOperatorVersionSplit[2]} >= ${minimumOperatorVersionSplit[2]}" 1962 echo "Returning False..:(" 1963 return false 1964 } 1965 1966 1967 def downloadCLI() { 1968 script { 1969 sh "echo Downloading VZ CLI from object storage" 1970 if(VERSION_FOR_UPGRADE_ENV == "master"){ 1971 sh "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name master/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}" 1972 }else{ 1973 sh "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}" 1974 } 1975 sh """ 1976 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 1977 ${GO_REPO_PATH}/vz version 1978 """ 1979 } 1980 } 1981 1982 def setInitValues() { 1983 script { 1984 def props = readProperties file: '.verrazzano-development-version' 1985 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 1986 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 1987 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 1988 // GIT_COMMIT_TO_USE: contains release branch sha which is used to download the release operator for the tip of a release branch. 1989 // If condition is executed only if the upgrade target branch is master. 1990 if(params.VERSION_FOR_UPGRADE == 'master'){ 1991 SHORT_COMMIT_HASH = params.GIT_COMMIT_TO_USE[0..7] 1992 } 1993 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 1994 // update the description with some meaningful info 1995 setDisplayName() 1996 currentBuild.description = "from ${params.VERSION_FOR_INSTALL}\n${SHORT_COMMIT_HASH} : ${env.GIT_COMMIT} : ${params.GIT_COMMIT_TO_USE}" 1997 if (params.TEST_ENV != "KIND") { 1998 // derive the prefix for the OKE cluster 1999 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${GO_REPO_PATH}/verrazzano/ci/scripts/derive_oke_cluster_name.sh").trim() 2000 } 2001 // Sets the VZ installation configuration based on VZ_INSTALL_CONFIG parameter. 2002 setVZConfigForInstallation() 2003 } 2004 } 2005 2006 def performDockerLogin() { 2007 script { 2008 try { 2009 sh """ 2010 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 2011 """ 2012 } catch(error) { 2013 echo "docker login failed, retrying after sleep" 2014 retry(4) { 2015 sleep(30) 2016 sh """ 2017 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 2018 """ 2019 } 2020 } 2021 } 2022 } 2023 2024 def performSourceCodeCheckout() { 2025 script { 2026 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 2027 if (params.GIT_COMMIT_TO_USE == "NONE") { 2028 echo "Specific GIT commit was not specified, use current head" 2029 def scmInfo = checkout scm 2030 env.GIT_COMMIT = scmInfo.GIT_COMMIT 2031 env.GIT_BRANCH = scmInfo.GIT_BRANCH 2032 // The master branch is checked out specifically when the upgrade target branch is master. 2033 // This is done to ensure the master branch upgrade test suites are run after the upgrade is completed. 2034 // Since the target upgrade branch is the master, then the tests run after upgrade must also be from the master branch. 2035 // This is here to ensure that happens. 2036 }else if(params.VERSION_FOR_UPGRADE == 'master'){ 2037 def scmInfo = checkout([ 2038 $class: 'GitSCM', 2039 branches: [[name: 'master']], 2040 doGenerateSubmoduleConfigurations: false, 2041 extensions: [], 2042 submoduleCfg: [], 2043 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 2044 env.GIT_COMMIT = scmInfo.GIT_COMMIT 2045 env.GIT_BRANCH = scmInfo.GIT_BRANCH 2046 }else { 2047 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 2048 def scmInfo = checkout([ 2049 $class: 'GitSCM', 2050 branches: [[name: params.GIT_COMMIT_TO_USE]], 2051 doGenerateSubmoduleConfigurations: false, 2052 extensions: [], 2053 submoduleCfg: [], 2054 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 2055 env.GIT_COMMIT = scmInfo.GIT_COMMIT 2056 env.GIT_BRANCH = scmInfo.GIT_BRANCH 2057 // If the commit we were handed is not what the SCM says we are using, fail 2058 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 2059 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 2060 exit 1 2061 } 2062 } 2063 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 2064 } 2065 } 2066 2067 def pipelinePostFailure() { 2068 script { 2069 sh """ 2070 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 2071 """ 2072 archiveArtifacts artifacts: '**/build-console-output.log,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true 2073 // Ignore failures in any of the following actions so that the "always" post step that cleans up clusters is executed 2074 sh """ 2075 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip || true 2076 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip || true 2077 rm archive.zip || true 2078 """ 2079 } 2080 } 2081 2082 def performDescribeVerrazzanoResource() { 2083 script { 2084 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 2085 for(int count=1; count<=clusterCount; count++) { 2086 sh """ 2087 echo "Performing describe on Verrazzano resource on cluster ${count}" 2088 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 2089 kubectl get vz -o yaml 2090 kubectl get features || true 2091 """ 2092 } 2093 } 2094 }