github.com/verrazzano/verrazzano@v1.7.1/ci/kind/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          timeout(time: 90, unit: 'MINUTES')
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16         docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "${agentLabel}"
    22          }
    23      }
    24  
    25      parameters {
    26          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    27                  description: 'Kubernetes Version for KinD Cluster',
    28                  // 1st choice is the default value
    29                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    30          string (name: 'GIT_COMMIT_TO_USE',
    31                          defaultValue: 'NONE',
    32                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    33                          trim: true)
    34          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    35                          defaultValue: 'NONE',
    36                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    37                          trim: true)
    38          choice (name: 'WILDCARD_DNS_DOMAIN',
    39                  description: 'This is the wildcard DNS domain',
    40                  // 1st choice is the default value
    41                  choices: [ "nip.io", "sslip.io"])
    42          choice (name: 'CRD_API_VERSION',
    43                  description: 'This is the API crd version.',
    44                  // 1st choice is the default value
    45                  choices: [ "v1beta1", "v1alpha1"])
    46          booleanParam (description: 'Whether to install Verrazzano using examples/ha/ha.yaml and 3 kind nodes (defaults to false)', name: 'INSTALL_HA', defaultValue: false)
    47          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    48          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    49          booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false)
    50          string (name: 'CONSOLE_REPO_BRANCH',
    51                  defaultValue: '',
    52                  description: 'The branch to check out after cloning the console repository.',
    53                  trim: true)
    54          booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    55          string (name: 'TAGGED_TESTS',
    56                  defaultValue: '',
    57                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    58                  trim: true)
    59          string (name: 'INCLUDED_TESTS',
    60                  defaultValue: '.*',
    61                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    62                  trim: true)
    63          string (name: 'EXCLUDED_TESTS',
    64                  defaultValue: '_excluded_test',
    65                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    66                  trim: true)
    67          booleanParam (description: 'Whether to run JWT tests', name: 'ENABLE_JWT_TESTING', defaultValue: false)
    68          booleanParam (description: 'Whether to run slow tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    69          booleanParam (description: 'Whether to run opensearch-operator based opensearch tests', name: 'RUN_OPENSEARCH_OPERATOR_TESTS', defaultValue: false)
    70          booleanParam (description: 'Whether to run clusterAPI override tests', name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', defaultValue: false)
    71          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    72          booleanParam (description: 'Whether to enable local IdP Dex', name: 'ENABLE_DEX', defaultValue: false)
    73      }
    74  
    75      environment {
    76          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    77          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    78          GOPATH = '/home/opc/go'
    79          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    80          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    81          DOCKER_EMAIL = credentials('github-packages-email')
    82          DOCKER_REPO = 'ghcr.io'
    83          DOCKER_NAMESPACE = 'verrazzano'
    84          NETRC_FILE = credentials('netrc')
    85          CLUSTER_NAME = 'verrazzano'
    86          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    87          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    88          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    89          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    90          OCR_CREDS = credentials('ocr-pull-and-push-account')
    91          OCR_REPO = 'container-registry.oracle.com'
    92          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    93          INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    94          INSTALL_CONFIG_FILE_KIND_HA = "./examples/ha/ha.yaml"
    95          INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml"
    96          INSTALL_PROFILE = "${params.INSTALL_HA == true ? "prod" : "dev"}"
    97          KIND_NODE_COUNT = "${params.INSTALL_HA == true ? "3" : "1"}"
    98          VZ_ENVIRONMENT_NAME = "default"
    99          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   100          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
   101          ENABLE_THANOS_STORE_GATEWAY = true
   102          ENABLE_THANOS_COMPACTOR = true
   103          ENABLE_THANOS_RULER = true
   104  
   105          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
   106          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
   107  
   108          // Environment variables required to capture cluster snapshot and bug report on test failure
   109          DUMP_KUBECONFIG="${KUBECONFIG}"
   110          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   111          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   112          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   113  
   114          // Environment variable for Verrazzano CLI executable
   115          VZ_COMMAND="${GO_REPO_PATH}/vz"
   116  
   117          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   118          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   119  
   120          // used for console artifact capture on failure
   121          JENKINS_READ = credentials('jenkins-auditor')
   122          OCI_CLI_AUTH="instance_principal"
   123          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   124          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   125          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   126  
   127          // used to emit metrics from Ginkgo suites
   128          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   129          TEST_ENV_LABEL = "kind"
   130          TEST_ENV = "KIND"
   131          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   132  
   133          // used to generate Ginkgo test reports
   134          TEST_REPORT = "test-report.xml"
   135          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   136          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   137  
   138          ENABLE_DEX = "${params.ENABLE_DEX == true ? true : false}"
   139      }
   140  
   141      stages {
   142          stage('Clean workspace and checkout') {
   143              steps {
   144                  sh """
   145                      echo "${NODE_LABELS}"
   146                  """
   147  
   148                  script {
   149                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   150                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   151                          echo "Specific GIT commit was not specified, use current head"
   152                          def scmInfo = checkout scm
   153                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   154                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   155                      } else {
   156                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   157                          def scmInfo = checkout([
   158                              $class: 'GitSCM',
   159                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   160                              doGenerateSubmoduleConfigurations: false,
   161                              extensions: [],
   162                              submoduleCfg: [],
   163                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   164                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   165                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   166                          // If the commit we were handed is not what the SCM says we are using, fail
   167                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   168                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   169                              exit 1
   170                          }
   171                      }
   172                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   173                  }
   174  
   175                  sh """
   176                      cp -f "${NETRC_FILE}" $HOME/.netrc
   177                      chmod 600 $HOME/.netrc
   178                  """
   179  
   180                  performDockerLogin()
   181  
   182                  sh """
   183                      rm -rf ${GO_REPO_PATH}/verrazzano
   184                      mkdir -p ${GO_REPO_PATH}/verrazzano
   185                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   186                  """
   187  
   188                  script {
   189                      def props = readProperties file: '.verrazzano-development-version'
   190                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   191                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   192                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   193                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   194                      // update the description with some meaningful info
   195                      setDisplayName()
   196                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   197                  }
   198                  script {
   199                      sh """
   200                          echo "Downloading VZ CLI from object storage"
   201                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   202                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   203                          ${GO_REPO_PATH}/vz version
   204                      """
   205                  }
   206              }
   207          }
   208  
   209          stage('Acceptance Tests') {
   210              stages {
   211                  stage('Prepare AT environment') {
   212                      environment {
   213                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   214                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   215                          REALM_USER_PASSWORD = credentials('todo-mysql-password')
   216                          REALM_NAME = "test-realm"
   217                          GITHUB_TOKEN = credentials('github-api-token-release-process')
   218                      }
   219                      steps {
   220                          script {
   221                              if (params.INSTALL_HA) {
   222                                  sh """
   223                                      cp ${env.INSTALL_CONFIG_FILE_KIND_HA} ${env.INSTALL_CONFIG_FILE_KIND}
   224                                      cd ${GO_REPO_PATH}/verrazzano
   225                                      ci/scripts/prepare_ha_test_at_environment.sh ${env.INSTALL_CONFIG_FILE_KIND}
   226                                  """
   227                              } else {
   228                                  sh """
   229                                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   230                                  """
   231                              }
   232                          }
   233                          sh """
   234                              cd ${GO_REPO_PATH}/verrazzano
   235                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA}
   236                          """
   237                          script {
   238                              if (params.ENABLE_JWT_TESTING) {
   239                                  sh """
   240                                      # setup test realm for JWT testing scenarios
   241                                      keycloakPassword=\$(kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo)
   242                                      sed -i "s|##KEYCLOAK_PASSWORD##|\$keycloakPassword|g" ci/scripts/create_test_realm.sh
   243                                      sed -i "s|##REALM_USER_PASSWORD##|${env.REALM_USER_PASSWORD}|g" ci/scripts/create_test_realm.sh
   244                                      sed -i "s|##REALM_NAME##|${env.REALM_NAME}|g" ci/scripts/create_test_realm.sh
   245                                      kubectl exec keycloak-0 -n keycloak -- /bin/sh -c "`cat ci/scripts/create_test_realm.sh`"
   246                                      # setup request authentication policy
   247                                      keycloakURI=\$(kubectl get ingress -n keycloak keycloak -o jsonpath="{.spec.rules[0].host}")
   248                                      sed -i "s|##KEYCLOAK_URI##|\$keycloakURI|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   249                                      jwks=\$(curl -sk https://\$keycloakURI/auth/realms/test-realm/protocol/openid-connect/certs | jq -c 'del(.keys[] | select( .use == "enc"))')
   250                                      sed -i "s|##JWKS_KEY##|\$jwks|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   251                                      kubectl apply -f tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   252                                  """
   253                              }
   254                          }
   255                      }
   256                      post {
   257                          failure {
   258                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   259                          }
   260                          always {
   261                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   262                              // enable debug logging of Verrazzano api istio proxy
   263                              script {
   264                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   265                                      sh '''
   266                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\")
   267                                          if [ -z "\$vz_api_pod" ]; then
   268                                            echo "Could not find verrazzano-authproxy pod, not enabling debug logging"
   269                                          else
   270                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   271                                          fi
   272                                          nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   273                                          if [ -z "\$nginx_ing_pod" ]; then
   274                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   275                                          else
   276                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   277                                          fi
   278                                      '''
   279                                  }
   280                              }
   281                          }
   282                      }
   283                  }
   284  
   285                  stage('Verify Install') {
   286                      environment {
   287                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install"
   288                      }
   289                      steps {
   290                          runGinkgoRandomize('verify-install')
   291                      }
   292                      post {
   293                          always {
   294                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   295                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   296                          }
   297                      }
   298                  }
   299  
   300                  stage ('console') {
   301                      environment {
   302                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console"
   303                      }
   304                      steps {
   305                          catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   306                              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   307                          }
   308                      }
   309                      post {
   310                          always {
   311                              sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   312                          }
   313                      }
   314                  }
   315  
   316                  stage('Run Acceptance Tests Infra') {
   317                      parallel {
   318                          stage('verify-scripts') {
   319                              steps {
   320                                  runGinkgoRandomize('scripts', "${KUBECONFIG}")
   321                              }
   322                          }
   323                          stage('verify-infra restapi') {
   324                              environment {
   325                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-restapi"
   326                              }
   327                              steps {
   328                                  runGinkgoRandomize('verify-infra/restapi')
   329                              }
   330                          }
   331                          stage('verify-infra oam') {
   332                              environment {
   333                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-oam"
   334                              }
   335                              steps {
   336                                  runGinkgoRandomize('verify-infra/oam')
   337                              }
   338                          }
   339                          stage('verify-infra vmi') {
   340                              environment {
   341                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-vmi"
   342                              }
   343                              steps {
   344                                  runGinkgoRandomize('verify-infra/vmi')
   345                              }
   346                          }
   347                          stage('security rbac') {
   348                              environment {
   349                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/sec-role-based-access"
   350                              }
   351                              steps {
   352                                  runGinkgo('security/rbac')
   353                              }
   354                          }
   355                          stage('system component metrics') {
   356                              environment {
   357                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-component-metrics"
   358                              }
   359                              steps {
   360                                  runGinkgo('metrics/syscomponents')
   361                              }
   362                          }
   363                          stage ('grafana db') {
   364                              when {
   365                                  expression {params.USE_DB_FOR_GRAFANA == true}
   366                              }
   367                              environment {
   368                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/grafana-db"
   369                              }
   370                              steps {
   371                                  runGinkgo('grafana-db')
   372                              }
   373                          }
   374                          stage('opensearch logging') {
   375                              environment {
   376                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-logging"
   377                              }
   378                              steps {
   379                                  runGinkgo('logging/opensearch')
   380                              }
   381                          }
   382                          stage('system logging') {
   383                              environment {
   384                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-logging"
   385                              }
   386                              steps {
   387                                  runGinkgo('logging/system')
   388                              }
   389                          }
   390                      }
   391                      post {
   392                          always {
   393                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true
   394                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   395                          }
   396                      }
   397                  }
   398  
   399                  stage('Run Acceptance Tests Deployments Group 1') {
   400                      parallel {
   401                          stage('istio authorization policy') {
   402                              environment {
   403                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/istio-authz-policy"
   404                              }
   405                              steps {
   406                                  runGinkgo('istio/authz')
   407                              }
   408                          }
   409                          stage('security network policies') {
   410                              environment {
   411                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/netpol"
   412                              }
   413                              steps {
   414                                  script {
   415                                      if (params.CREATE_CLUSTER_USE_CALICO == true) {
   416                                          runGinkgo('security/netpol')
   417                                      }
   418                                  }
   419                              }
   420                          }
   421                          stage('deployment metrics') {
   422                              environment {
   423                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/k8sdeploy-workload-metrics"
   424                              }
   425                              steps {
   426                                  runGinkgo('metrics/deploymetrics')
   427                              }
   428                          }
   429                          stage('examples logging helidon') {
   430                              environment {
   431                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-logging-helidon"
   432                              }
   433                              steps {
   434                                  runGinkgo('logging/helidon')
   435                              }
   436                          }
   437                          stage('examples helidon') {
   438                              environment {
   439                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   440                              }
   441                              steps {
   442                                  runGinkgo('examples/helidon')
   443                              }
   444                          }
   445                          stage('examples helidon service template') {
   446                              environment {
   447                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-service-template"
   448                              }
   449                              steps {
   450                                  runGinkgoAppTest('examples/helidon', 'hello-helidon-service-template', "${DUMP_DIRECTORY}", 'false', 'false', 'examples/hello-helidon/hello-helidon-comp-service-template.yaml', 'examples/hello-helidon/hello-helidon-app-custom-port.yaml')
   451                              }
   452                          }
   453                          stage('examples helidon manualscalertait') {
   454                              environment {
   455                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-manualscalertait"
   456                              }
   457                              steps {
   458                                  runGinkgoAppTest('examples/helidon', 'examples-helidon-manualscalertait', "${DUMP_DIRECTORY}", 'false', 'false','examples/hello-helidon/hello-helidon-comp.yaml', 'examples/hello-helidon/hello-helidon-app-scaler-trait.yaml')
   459                              }
   460                          }
   461  
   462                          stage('jaeger helidon') {
   463                              environment {
   464                                 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jaeger-helidon"
   465                              }
   466                              steps {
   467                                 runGinkgo('jaeger/helidon')
   468                              }
   469                          }
   470                          stage('jaeger system') {
   471                              environment {
   472                                 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jaeger-system"
   473                              }
   474                              steps {
   475                                 runGinkgo('jaeger/system')
   476                              }
   477                          }
   478                          stage('jwt helidon') {
   479                              when {
   480                                  expression {params.ENABLE_JWT_TESTING == true}
   481                              }
   482                              environment {
   483                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jwt-helidon"
   484                                  REALM_USER_PASSWORD = credentials('todo-mysql-password')
   485                                  REALM_NAME="test-realm"
   486                              }
   487                              steps {
   488                                  runGinkgo('jwt/helidon')
   489                              }
   490                          }
   491                          stage('jwt helidon svc') {
   492                              when {
   493                                  expression {params.ENABLE_JWT_TESTING == true}
   494                              }
   495                              environment {
   496                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jwt-helidon-svc"
   497                                  REALM_USER_PASSWORD = credentials('todo-mysql-password')
   498                                  REALM_NAME="test-realm"
   499                              }
   500                              steps {
   501                                  runGinkgo('jwt/helidon-svc')
   502                              }
   503                          }
   504                          stage('examples helidon-metrics') {
   505                              environment {
   506                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidonmetrics"
   507                              }
   508                              when {
   509                                  expression {params.RUN_SLOW_TESTS == true}
   510                              }
   511                              steps {
   512                                  runGinkgo('examples/helidonmetrics')
   513                              }
   514                          }
   515                          stage('logging trait Helidon workload') {
   516                              environment {
   517                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-helidonworkload"
   518                              }
   519                              steps {
   520                                  runGinkgo('loggingtrait/helidonworkload')
   521                              }
   522                          }
   523                      }
   524                      post {
   525                          always {
   526                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true
   527                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   528                          }
   529                      }
   530                  }
   531  
   532                  stage('Ingress & logging trait WebLogic workload') {
   533                      environment {
   534                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-weblogicworkload"
   535                      }
   536                      steps {
   537                          runGinkgo('loggingtrait/weblogicworkload')
   538                      }
   539                  }
   540  
   541                  stage('Run Acceptance Tests Deployments Group 2') {
   542                      parallel {
   543                          stage('opensearch-topology') {
   544                              environment {
   545                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-topology"
   546                              }
   547                              steps {
   548                                  runGinkgoRandomize('opensearch/topology')
   549                              }
   550                          }
   551                          stage('weblogic workload') {
   552                              environment {
   553                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-workload"
   554                              }
   555                              steps {
   556                                  runGinkgo('workloads/weblogic')
   557                              }
   558                          }
   559                          stage('coherence workload') {
   560                              environment {
   561                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/coherence-workload"
   562                              }
   563                              steps {
   564                                  runGinkgo('workloads/coherence')
   565                              }
   566                          }
   567                          stage('oam workloads') {
   568                              environment {
   569                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/oam-workloads"
   570                              }
   571                              steps {
   572                                  runGinkgo('workloads/oam')
   573                              }
   574                          }
   575                          stage('logging trait Coherence workload') {
   576                              environment {
   577                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-coherenceworkload"
   578                              }
   579                              steps {
   580                                  runGinkgo('loggingtrait/coherenceworkload')
   581                              }
   582                          }
   583                      }
   584                  }
   585  
   586                  stage('Run OpenSearch Operator tests') {
   587                      when {
   588                          expression {params.RUN_OPENSEARCH_OPERATOR_TESTS == true}
   589                      }
   590                      stages {
   591                          stage('install and verify opensearch operator') {
   592                              environment {
   593                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-operator-install"
   594                              }
   595                              steps {
   596                                  runGinkgo('opensearch-operator/install')
   597                              }
   598                          }
   599                          stage('verify opensearch infra') {
   600                              environment {
   601                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-operator-infra"
   602                              }
   603                              steps {
   604                                  runGinkgo('opensearch-operator/infra')
   605                              }
   606                          }
   607                          stage('opensearch topology') {
   608                              environment {
   609                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-operator-topology"
   610                              }
   611                              steps {
   612                                  runGinkgo('opensearch-operator/topology')
   613                              }
   614                          }
   615                      }
   616                  }
   617  
   618                  stage('ClusterAPI Override Tests') {
   619                      when { expression {params.RUN_CLUSTERAPI_OVERRIDE_TESTS} }
   620                      environment {
   621                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/clusterapi-overrides"
   622                      }
   623                      steps {
   624                          runGinkgo('clusterapi/capi-overrides')
   625                      }
   626                      post {
   627                          failure {
   628                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   629                              dumpK8sCluster('clusterAPI-override-tests-dump')
   630                          }
   631                          always {
   632                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   633                          }
   634                      }
   635                  }
   636              }
   637  
   638              post {
   639                  aborted {
   640                      script {
   641                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   642                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   643                          }
   644                      }
   645                  }
   646                  failure {
   647                      script {
   648                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   649                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   650                          }
   651                      }
   652                  }
   653                  success {
   654                      script {
   655                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   656                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   657                          }
   658                      }
   659                  }
   660              }
   661          }
   662      }
   663  
   664      post {
   665          always {
   666              script {
   667                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   668                      dumpVerrazzanoSystemPods()
   669                      dumpCattleSystemPods()
   670                      dumpNginxIngressControllerLogs()
   671                      dumpVerrazzanoPlatformOperatorLogs()
   672                      dumpVerrazzanoApplicationOperatorLogs()
   673                      dumpOamKubernetesRuntimeLogs()
   674                      dumpVerrazzanoApiLogs()
   675                  }
   676              }
   677  
   678              sh """
   679                  # Copy the generated test reports to WORKSPACE to archive them
   680                  mkdir -p ${TEST_REPORT_DIR}
   681                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   682                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   683              """
   684              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   685              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   686              deleteCluster()
   687          }
   688          failure {
   689              sh """
   690                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   691              """
   692              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   693              sh """
   694                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   695                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   696                  rm archive.zip
   697              """
   698          }
   699          cleanup {
   700              deleteDir()
   701          }
   702      }
   703  }
   704  
   705  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   706      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   707          sh """
   708              if [ ! -z "${kubeConfig}" ]; then
   709                  export KUBECONFIG="${kubeConfig}"
   710              fi
   711              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   712              if [ -d "${testSuitePath}" ]; then
   713                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   714              fi
   715          """
   716      }
   717  }
   718  
   719  def runGinkgoAppTest(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false', component='', appConfig='') {
   720      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   721          sh """
   722              if [ ! -z "${dumpDir}" ]; then
   723                  export DUMP_DIRECTORY=${dumpDir}
   724              fi
   725              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   726              ginkgo -v --keep-going --no-color ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace} --component=${component} --appconfig=${appConfig}
   727          """
   728      }
   729  }
   730  
   731  def runSocksVariant(variant) {
   732      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   733          sh """
   734              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   735              SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}"  examples/socks/...
   736          """
   737      }
   738  }
   739  
   740  def runGinkgo(testSuitePath) {
   741      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   742          sh """
   743              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   744              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   745          """
   746      }
   747  }
   748  
   749  def dumpK8sCluster(dumpDirectory) {
   750      sh """
   751          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   752      """
   753  }
   754  
   755  def dumpVerrazzanoSystemPods() {
   756      sh """
   757          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   758          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   759          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   760          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   761          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   762          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   763          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   764          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   765          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   766      """
   767  }
   768  
   769  def dumpCattleSystemPods() {
   770      sh """
   771          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   772          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   773          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   774          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   775          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   776      """
   777  }
   778  
   779  def dumpNginxIngressControllerLogs() {
   780      sh """
   781          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   782          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   783          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   784      """
   785  }
   786  
   787  def dumpVerrazzanoPlatformOperatorLogs() {
   788      sh """
   789          ## dump out verrazzano-platform-operator logs
   790          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   791          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   792          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   793          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   794          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   795          echo "------------------------------------------"
   796      """
   797  }
   798  
   799  def dumpVerrazzanoApplicationOperatorLogs() {
   800      sh """
   801          ## dump out verrazzano-application-operator logs
   802          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   803          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   804          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   805          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   806          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   807          echo "------------------------------------------"
   808      """
   809  }
   810  
   811  def dumpOamKubernetesRuntimeLogs() {
   812      sh """
   813          ## dump out oam-kubernetes-runtime logs
   814          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   815          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   816          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   817          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   818          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   819          echo "------------------------------------------"
   820      """
   821  }
   822  
   823  def dumpVerrazzanoApiLogs() {
   824      sh """
   825          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   826          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   827          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   828      """
   829  }
   830  
   831  def getEffectiveDumpOnSuccess() {
   832      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   833      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   834          effectiveValue = true
   835          echo "Forcing dump on success based on global override setting"
   836      }
   837      return effectiveValue
   838  }
   839  
   840  def deleteCluster() {
   841      sh """
   842          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   843          make delete-cluster
   844          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   845            echo "Failures seen during dumping of artifacts, treat post as failed"
   846            exit 1
   847          fi
   848      """
   849  }
   850  
   851  def setDisplayName() {
   852      echo "Start setDisplayName"
   853      def causes = currentBuild.getBuildCauses()
   854      echo "causes: " + causes.toString()
   855      for (cause in causes) {
   856          def causeString = cause.toString()
   857          echo "current cause: " + causeString
   858          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   859               echo "This job was caused by " + causeString
   860               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   861                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   862               } else if (causeString.contains("verrazzano-flaky-tests")) {
   863                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   864               }
   865           }
   866      }
   867      echo "End setDisplayName"
   868  }
   869  
   870  def performDockerLogin() {
   871      script {
   872          try {
   873              sh """
   874                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   875              """
   876          } catch(error) {
   877              echo "docker login failed, retrying after sleep"
   878              retry(4) {
   879                  sleep(30)
   880                  sh """
   881                      echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   882                  """
   883              }
   884          }
   885      }
   886  }