github.com/verrazzano/verrazzano@v1.7.0/ci/kind/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          timeout(time: 90, unit: 'MINUTES')
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16         docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "${agentLabel}"
    22          }
    23      }
    24  
    25      parameters {
    26          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    27                  description: 'Kubernetes Version for KinD Cluster',
    28                  // 1st choice is the default value
    29                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    30          string (name: 'GIT_COMMIT_TO_USE',
    31                          defaultValue: 'NONE',
    32                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    33                          trim: true)
    34          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    35                          defaultValue: 'NONE',
    36                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    37                          trim: true)
    38          choice (name: 'WILDCARD_DNS_DOMAIN',
    39                  description: 'This is the wildcard DNS domain',
    40                  // 1st choice is the default value
    41                  choices: [ "nip.io", "sslip.io"])
    42          choice (name: 'CRD_API_VERSION',
    43                  description: 'This is the API crd version.',
    44                  // 1st choice is the default value
    45                  choices: [ "v1beta1", "v1alpha1"])
    46          booleanParam (description: 'Whether to install Verrazzano using examples/ha/ha.yaml and 3 kind nodes (defaults to false)', name: 'INSTALL_HA', defaultValue: false)
    47          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    48          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    49          booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false)
    50          string (name: 'CONSOLE_REPO_BRANCH',
    51                  defaultValue: '',
    52                  description: 'The branch to check out after cloning the console repository.',
    53                  trim: true)
    54          booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    55          string (name: 'TAGGED_TESTS',
    56                  defaultValue: '',
    57                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    58                  trim: true)
    59          string (name: 'INCLUDED_TESTS',
    60                  defaultValue: '.*',
    61                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    62                  trim: true)
    63          string (name: 'EXCLUDED_TESTS',
    64                  defaultValue: '_excluded_test',
    65                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    66                  trim: true)
    67          booleanParam (description: 'Whether to run JWT tests', name: 'ENABLE_JWT_TESTING', defaultValue: false)
    68          booleanParam (description: 'Whether to run slow tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    69          booleanParam (description: 'Whether to run opensearch-operator based opensearch tests', name: 'RUN_OPENSEARCH_OPERATOR_TESTS', defaultValue: false)
    70          booleanParam (description: 'Whether to run clusterAPI override tests', name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', defaultValue: false)
    71          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    72          booleanParam (description: 'Whether to enable local IdP Dex', name: 'ENABLE_DEX', defaultValue: false)
    73      }
    74  
    75      environment {
    76          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    77          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    78          GOPATH = '/home/opc/go'
    79          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    80          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    81          DOCKER_EMAIL = credentials('github-packages-email')
    82          DOCKER_REPO = 'ghcr.io'
    83          DOCKER_NAMESPACE = 'verrazzano'
    84          NETRC_FILE = credentials('netrc')
    85          CLUSTER_NAME = 'verrazzano'
    86          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    87          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    88          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    89          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    90          OCR_CREDS = credentials('ocr-pull-and-push-account')
    91          OCR_REPO = 'container-registry.oracle.com'
    92          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    93          INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    94          INSTALL_CONFIG_FILE_KIND_HA = "./examples/ha/ha.yaml"
    95          INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml"
    96          INSTALL_PROFILE = "${params.INSTALL_HA == true ? "prod" : "dev"}"
    97          KIND_NODE_COUNT = "${params.INSTALL_HA == true ? "3" : "1"}"
    98          VZ_ENVIRONMENT_NAME = "default"
    99          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   100          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
   101          ENABLE_THANOS_STORE_GATEWAY = true
   102          ENABLE_THANOS_COMPACTOR = true
   103          ENABLE_THANOS_RULER = true
   104  
   105          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
   106          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
   107  
   108          // Environment variables required to capture cluster snapshot and bug report on test failure
   109          DUMP_KUBECONFIG="${KUBECONFIG}"
   110          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   111          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   112          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   113  
   114          // Environment variable for Verrazzano CLI executable
   115          VZ_COMMAND="${GO_REPO_PATH}/vz"
   116  
   117          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   118          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   119  
   120          // used for console artifact capture on failure
   121          JENKINS_READ = credentials('jenkins-auditor')
   122          OCI_CLI_AUTH="instance_principal"
   123          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   124          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   125          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   126          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   127  
   128          // used to emit metrics from Ginkgo suites
   129          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   130          TEST_ENV_LABEL = "kind"
   131          TEST_ENV = "KIND"
   132          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   133          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   134          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   135          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   136  
   137          // used to generate Ginkgo test reports
   138          TEST_REPORT = "test-report.xml"
   139          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   140          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   141  
   142          ENABLE_DEX = "${params.ENABLE_DEX == true ? true : false}"
   143      }
   144  
   145      stages {
   146          stage('Clean workspace and checkout') {
   147              steps {
   148                  sh """
   149                      echo "${NODE_LABELS}"
   150                  """
   151  
   152                  script {
   153                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   154                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   155                          echo "Specific GIT commit was not specified, use current head"
   156                          def scmInfo = checkout scm
   157                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   158                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   159                      } else {
   160                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   161                          def scmInfo = checkout([
   162                              $class: 'GitSCM',
   163                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   164                              doGenerateSubmoduleConfigurations: false,
   165                              extensions: [],
   166                              submoduleCfg: [],
   167                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   168                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   169                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   170                          // If the commit we were handed is not what the SCM says we are using, fail
   171                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   172                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   173                              exit 1
   174                          }
   175                      }
   176                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   177                  }
   178  
   179                  sh """
   180                      cp -f "${NETRC_FILE}" $HOME/.netrc
   181                      chmod 600 $HOME/.netrc
   182                  """
   183  
   184                  performDockerLogin()
   185  
   186                  sh """
   187                      rm -rf ${GO_REPO_PATH}/verrazzano
   188                      mkdir -p ${GO_REPO_PATH}/verrazzano
   189                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   190                  """
   191  
   192                  script {
   193                      def props = readProperties file: '.verrazzano-development-version'
   194                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   195                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   196                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   197                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   198                      // update the description with some meaningful info
   199                      setDisplayName()
   200                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   201                  }
   202                  script {
   203                      sh """
   204                          echo "Downloading VZ CLI from object storage"
   205                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   206                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   207                          ${GO_REPO_PATH}/vz version
   208                      """
   209                  }
   210              }
   211          }
   212  
   213          stage('Acceptance Tests') {
   214              stages {
   215                  stage('Prepare AT environment') {
   216                      environment {
   217                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   218                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   219                          REALM_USER_PASSWORD = credentials('todo-mysql-password')
   220                          REALM_NAME = "test-realm"
   221                          GITHUB_TOKEN = credentials('github-api-token-release-process')
   222                      }
   223                      steps {
   224                          script {
   225                              if (params.INSTALL_HA) {
   226                                  sh """
   227                                      cp ${env.INSTALL_CONFIG_FILE_KIND_HA} ${env.INSTALL_CONFIG_FILE_KIND}
   228                                      cd ${GO_REPO_PATH}/verrazzano
   229                                      ci/scripts/prepare_ha_test_at_environment.sh ${env.INSTALL_CONFIG_FILE_KIND}
   230                                  """
   231                              } else {
   232                                  sh """
   233                                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   234                                  """
   235                              }
   236                          }
   237                          sh """
   238                              cd ${GO_REPO_PATH}/verrazzano
   239                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA}
   240                          """
   241                          script {
   242                              if (params.ENABLE_JWT_TESTING) {
   243                                  sh """
   244                                      # setup test realm for JWT testing scenarios
   245                                      keycloakPassword=\$(kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo)
   246                                      sed -i "s|##KEYCLOAK_PASSWORD##|\$keycloakPassword|g" ci/scripts/create_test_realm.sh
   247                                      sed -i "s|##REALM_USER_PASSWORD##|${env.REALM_USER_PASSWORD}|g" ci/scripts/create_test_realm.sh
   248                                      sed -i "s|##REALM_NAME##|${env.REALM_NAME}|g" ci/scripts/create_test_realm.sh
   249                                      kubectl exec keycloak-0 -n keycloak -- /bin/sh -c "`cat ci/scripts/create_test_realm.sh`"
   250                                      # setup request authentication policy
   251                                      keycloakURI=\$(kubectl get ingress -n keycloak keycloak -o jsonpath="{.spec.rules[0].host}")
   252                                      sed -i "s|##KEYCLOAK_URI##|\$keycloakURI|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   253                                      jwks=\$(curl -sk https://\$keycloakURI/auth/realms/test-realm/protocol/openid-connect/certs | jq -c 'del(.keys[] | select( .use == "enc"))')
   254                                      sed -i "s|##JWKS_KEY##|\$jwks|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   255                                      kubectl apply -f tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   256                                  """
   257                              }
   258                          }
   259                      }
   260                      post {
   261                          failure {
   262                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   263                          }
   264                          always {
   265                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   266                              // enable debug logging of Verrazzano api istio proxy
   267                              script {
   268                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   269                                      sh '''
   270                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\")
   271                                          if [ -z "\$vz_api_pod" ]; then
   272                                            echo "Could not find verrazzano-authproxy pod, not enabling debug logging"
   273                                          else
   274                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   275                                          fi
   276                                          nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   277                                          if [ -z "\$nginx_ing_pod" ]; then
   278                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   279                                          else
   280                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   281                                          fi
   282                                      '''
   283                                  }
   284                              }
   285                          }
   286                      }
   287                  }
   288  
   289                  stage('Verify Install') {
   290                      environment {
   291                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install"
   292                      }
   293                      steps {
   294                          runGinkgoRandomize('verify-install')
   295                      }
   296                      post {
   297                          always {
   298                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   299                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   300                          }
   301                      }
   302                  }
   303  
   304                  stage ('console') {
   305                      environment {
   306                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console"
   307                      }
   308                      steps {
   309                          catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   310                              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   311                          }
   312                      }
   313                      post {
   314                          always {
   315                              sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   316                          }
   317                      }
   318                  }
   319  
   320                  stage('Run Acceptance Tests Infra') {
   321                      parallel {
   322                          stage('verify-scripts') {
   323                              steps {
   324                                  runGinkgoRandomize('scripts', "${KUBECONFIG}")
   325                              }
   326                          }
   327                          stage('verify-infra restapi') {
   328                              environment {
   329                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-restapi"
   330                              }
   331                              steps {
   332                                  runGinkgoRandomize('verify-infra/restapi')
   333                              }
   334                          }
   335                          stage('verify-infra oam') {
   336                              environment {
   337                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-oam"
   338                              }
   339                              steps {
   340                                  runGinkgoRandomize('verify-infra/oam')
   341                              }
   342                          }
   343                          stage('verify-infra vmi') {
   344                              environment {
   345                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-vmi"
   346                              }
   347                              steps {
   348                                  runGinkgoRandomize('verify-infra/vmi')
   349                              }
   350                          }
   351                          stage('security rbac') {
   352                              environment {
   353                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/sec-role-based-access"
   354                              }
   355                              steps {
   356                                  runGinkgo('security/rbac')
   357                              }
   358                          }
   359                          stage('system component metrics') {
   360                              environment {
   361                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-component-metrics"
   362                              }
   363                              steps {
   364                                  runGinkgo('metrics/syscomponents')
   365                              }
   366                          }
   367                          stage ('grafana db') {
   368                              when {
   369                                  expression {params.USE_DB_FOR_GRAFANA == true}
   370                              }
   371                              environment {
   372                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/grafana-db"
   373                              }
   374                              steps {
   375                                  runGinkgo('grafana-db')
   376                              }
   377                          }
   378                          stage('opensearch logging') {
   379                              environment {
   380                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-logging"
   381                              }
   382                              steps {
   383                                  runGinkgo('logging/opensearch')
   384                              }
   385                          }
   386                          stage('system logging') {
   387                              environment {
   388                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-logging"
   389                              }
   390                              steps {
   391                                  runGinkgo('logging/system')
   392                              }
   393                          }
   394                      }
   395                      post {
   396                          always {
   397                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true
   398                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   399                          }
   400                      }
   401                  }
   402  
   403                  stage('Run Acceptance Tests Deployments Group 1') {
   404                      parallel {
   405                          stage('istio authorization policy') {
   406                              environment {
   407                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/istio-authz-policy"
   408                              }
   409                              steps {
   410                                  runGinkgo('istio/authz')
   411                              }
   412                          }
   413                          stage('security network policies') {
   414                              environment {
   415                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/netpol"
   416                              }
   417                              steps {
   418                                  script {
   419                                      if (params.CREATE_CLUSTER_USE_CALICO == true) {
   420                                          runGinkgo('security/netpol')
   421                                      }
   422                                  }
   423                              }
   424                          }
   425                          stage('deployment metrics') {
   426                              environment {
   427                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/k8sdeploy-workload-metrics"
   428                              }
   429                              steps {
   430                                  runGinkgo('metrics/deploymetrics')
   431                              }
   432                          }
   433                          stage('examples logging helidon') {
   434                              environment {
   435                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-logging-helidon"
   436                              }
   437                              steps {
   438                                  runGinkgo('logging/helidon')
   439                              }
   440                          }
   441                          stage('examples helidon') {
   442                              environment {
   443                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   444                              }
   445                              steps {
   446                                  runGinkgo('examples/helidon')
   447                              }
   448                          }
   449                          stage('examples helidon service template') {
   450                              environment {
   451                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-service-template"
   452                              }
   453                              steps {
   454                                  runGinkgoAppTest('examples/helidon', 'hello-helidon-service-template', "${DUMP_DIRECTORY}", 'false', 'false', 'examples/hello-helidon/hello-helidon-comp-service-template.yaml', 'examples/hello-helidon/hello-helidon-app-custom-port.yaml')
   455                              }
   456                          }
   457                          stage('examples helidon manualscalertait') {
   458                              environment {
   459                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-manualscalertait"
   460                              }
   461                              steps {
   462                                  runGinkgoAppTest('examples/helidon', 'examples-helidon-manualscalertait', "${DUMP_DIRECTORY}", 'false', 'false','examples/hello-helidon/hello-helidon-comp.yaml', 'examples/hello-helidon/hello-helidon-app-scaler-trait.yaml')
   463                              }
   464                          }
   465  
   466                          stage('jaeger helidon') {
   467                              environment {
   468                                 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jaeger-helidon"
   469                              }
   470                              steps {
   471                                 runGinkgo('jaeger/helidon')
   472                              }
   473                          }
   474                          stage('jaeger system') {
   475                              environment {
   476                                 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jaeger-system"
   477                              }
   478                              steps {
   479                                 runGinkgo('jaeger/system')
   480                              }
   481                          }
   482                          stage('jwt helidon') {
   483                              when {
   484                                  expression {params.ENABLE_JWT_TESTING == true}
   485                              }
   486                              environment {
   487                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jwt-helidon"
   488                                  REALM_USER_PASSWORD = credentials('todo-mysql-password')
   489                                  REALM_NAME="test-realm"
   490                              }
   491                              steps {
   492                                  runGinkgo('jwt/helidon')
   493                              }
   494                          }
   495                          stage('jwt helidon svc') {
   496                              when {
   497                                  expression {params.ENABLE_JWT_TESTING == true}
   498                              }
   499                              environment {
   500                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jwt-helidon-svc"
   501                                  REALM_USER_PASSWORD = credentials('todo-mysql-password')
   502                                  REALM_NAME="test-realm"
   503                              }
   504                              steps {
   505                                  runGinkgo('jwt/helidon-svc')
   506                              }
   507                          }
   508                          stage('examples helidon-metrics') {
   509                              environment {
   510                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidonmetrics"
   511                              }
   512                              when {
   513                                  expression {params.RUN_SLOW_TESTS == true}
   514                              }
   515                              steps {
   516                                  runGinkgo('examples/helidonmetrics')
   517                              }
   518                          }
   519                          stage('logging trait Helidon workload') {
   520                              environment {
   521                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-helidonworkload"
   522                              }
   523                              steps {
   524                                  runGinkgo('loggingtrait/helidonworkload')
   525                              }
   526                          }
   527                      }
   528                      post {
   529                          always {
   530                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true
   531                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   532                          }
   533                      }
   534                  }
   535  
   536                  stage('Ingress & logging trait WebLogic workload') {
   537                      environment {
   538                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-weblogicworkload"
   539                      }
   540                      steps {
   541                          runGinkgo('loggingtrait/weblogicworkload')
   542                      }
   543                  }
   544  
   545                  stage('Run Acceptance Tests Deployments Group 2') {
   546                      parallel {
   547                          stage('opensearch-topology') {
   548                              environment {
   549                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-topology"
   550                              }
   551                              steps {
   552                                  runGinkgoRandomize('opensearch/topology')
   553                              }
   554                          }
   555                          stage('weblogic workload') {
   556                              environment {
   557                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-workload"
   558                              }
   559                              steps {
   560                                  runGinkgo('workloads/weblogic')
   561                              }
   562                          }
   563                          stage('coherence workload') {
   564                              environment {
   565                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/coherence-workload"
   566                              }
   567                              steps {
   568                                  runGinkgo('workloads/coherence')
   569                              }
   570                          }
   571                          stage('oam workloads') {
   572                              environment {
   573                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/oam-workloads"
   574                              }
   575                              steps {
   576                                  runGinkgo('workloads/oam')
   577                              }
   578                          }
   579                          stage('logging trait Coherence workload') {
   580                              environment {
   581                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-coherenceworkload"
   582                              }
   583                              steps {
   584                                  runGinkgo('loggingtrait/coherenceworkload')
   585                              }
   586                          }
   587                      }
   588                  }
   589  
   590                  stage('Run OpenSearch Operator tests') {
   591                      when {
   592                          expression {params.RUN_OPENSEARCH_OPERATOR_TESTS == true}
   593                      }
   594                      stages {
   595                          stage('install and verify opensearch operator') {
   596                              environment {
   597                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-operator-install"
   598                              }
   599                              steps {
   600                                  runGinkgo('opensearch-operator/install')
   601                              }
   602                          }
   603                          stage('verify opensearch infra') {
   604                              environment {
   605                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-operator-infra"
   606                              }
   607                              steps {
   608                                  runGinkgo('opensearch-operator/infra')
   609                              }
   610                          }
   611                          stage('opensearch topology') {
   612                              environment {
   613                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-operator-topology"
   614                              }
   615                              steps {
   616                                  runGinkgo('opensearch-operator/topology')
   617                              }
   618                          }
   619                      }
   620                  }
   621  
   622                  stage('ClusterAPI Override Tests') {
   623                      when { expression {params.RUN_CLUSTERAPI_OVERRIDE_TESTS} }
   624                      environment {
   625                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/clusterapi-overrides"
   626                      }
   627                      steps {
   628                          runGinkgo('clusterapi/capi-overrides')
   629                      }
   630                      post {
   631                          failure {
   632                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   633                              dumpK8sCluster('clusterAPI-override-tests-dump')
   634                          }
   635                          always {
   636                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   637                          }
   638                      }
   639                  }
   640              }
   641  
   642              post {
   643                  aborted {
   644                      script {
   645                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   646                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   647                          }
   648                      }
   649                  }
   650                  failure {
   651                      script {
   652                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   653                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   654                          }
   655                      }
   656                  }
   657                  success {
   658                      script {
   659                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   660                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   661                          }
   662                      }
   663                  }
   664              }
   665          }
   666      }
   667  
   668      post {
   669          always {
   670              script {
   671                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   672                      dumpVerrazzanoSystemPods()
   673                      dumpCattleSystemPods()
   674                      dumpNginxIngressControllerLogs()
   675                      dumpVerrazzanoPlatformOperatorLogs()
   676                      dumpVerrazzanoApplicationOperatorLogs()
   677                      dumpOamKubernetesRuntimeLogs()
   678                      dumpVerrazzanoApiLogs()
   679                  }
   680              }
   681  
   682              sh """
   683                  # Copy the generated test reports to WORKSPACE to archive them
   684                  mkdir -p ${TEST_REPORT_DIR}
   685                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   686                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   687              """
   688              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   689              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   690              deleteCluster()
   691          }
   692          failure {
   693              sh """
   694                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   695              """
   696              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   697              sh """
   698                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   699                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   700                  rm archive.zip
   701              """
   702          }
   703          cleanup {
   704              deleteDir()
   705          }
   706      }
   707  }
   708  
   709  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   710      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   711          sh """
   712              if [ ! -z "${kubeConfig}" ]; then
   713                  export KUBECONFIG="${kubeConfig}"
   714              fi
   715              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   716              if [ -d "${testSuitePath}" ]; then
   717                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   718              fi
   719          """
   720      }
   721  }
   722  
   723  def runGinkgoAppTest(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false', component='', appConfig='') {
   724      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   725          sh """
   726              if [ ! -z "${dumpDir}" ]; then
   727                  export DUMP_DIRECTORY=${dumpDir}
   728              fi
   729              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   730              ginkgo -v --keep-going --no-color ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace} --component=${component} --appconfig=${appConfig}
   731          """
   732      }
   733  }
   734  
   735  def runSocksVariant(variant) {
   736      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   737          sh """
   738              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   739              SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}"  examples/socks/...
   740          """
   741      }
   742  }
   743  
   744  def runGinkgo(testSuitePath) {
   745      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   746          sh """
   747              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   748              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   749          """
   750      }
   751  }
   752  
   753  def dumpK8sCluster(dumpDirectory) {
   754      sh """
   755          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   756      """
   757  }
   758  
   759  def dumpVerrazzanoSystemPods() {
   760      sh """
   761          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   762          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   763          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   764          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   765          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   766          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   767          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   768          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   769          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   770      """
   771  }
   772  
   773  def dumpCattleSystemPods() {
   774      sh """
   775          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   776          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   777          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   778          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   779          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   780      """
   781  }
   782  
   783  def dumpNginxIngressControllerLogs() {
   784      sh """
   785          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   786          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   787          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   788      """
   789  }
   790  
   791  def dumpVerrazzanoPlatformOperatorLogs() {
   792      sh """
   793          ## dump out verrazzano-platform-operator logs
   794          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   795          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   796          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   797          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   798          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   799          echo "------------------------------------------"
   800      """
   801  }
   802  
   803  def dumpVerrazzanoApplicationOperatorLogs() {
   804      sh """
   805          ## dump out verrazzano-application-operator logs
   806          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   807          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   808          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   809          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   810          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   811          echo "------------------------------------------"
   812      """
   813  }
   814  
   815  def dumpOamKubernetesRuntimeLogs() {
   816      sh """
   817          ## dump out oam-kubernetes-runtime logs
   818          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   819          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   820          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   821          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   822          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   823          echo "------------------------------------------"
   824      """
   825  }
   826  
   827  def dumpVerrazzanoApiLogs() {
   828      sh """
   829          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   830          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   831          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   832      """
   833  }
   834  
   835  def getEffectiveDumpOnSuccess() {
   836      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   837      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   838          effectiveValue = true
   839          echo "Forcing dump on success based on global override setting"
   840      }
   841      return effectiveValue
   842  }
   843  
   844  def deleteCluster() {
   845      sh """
   846          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   847          make delete-cluster
   848          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   849            echo "Failures seen during dumping of artifacts, treat post as failed"
   850            exit 1
   851          fi
   852      """
   853  }
   854  
   855  def setDisplayName() {
   856      echo "Start setDisplayName"
   857      def causes = currentBuild.getBuildCauses()
   858      echo "causes: " + causes.toString()
   859      for (cause in causes) {
   860          def causeString = cause.toString()
   861          echo "current cause: " + causeString
   862          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   863               echo "This job was caused by " + causeString
   864               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   865                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   866               } else if (causeString.contains("verrazzano-flaky-tests")) {
   867                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   868               }
   869           }
   870      }
   871      echo "End setDisplayName"
   872  }
   873  
   874  def performDockerLogin() {
   875      script {
   876          try {
   877              sh """
   878                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   879              """
   880          } catch(error) {
   881              echo "docker login failed, retrying after sleep"
   882              retry(4) {
   883                  sleep(30)
   884                  sh """
   885                      echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   886                  """
   887              }
   888          }
   889      }
   890  }