github.com/verrazzano/verrazzano@v1.7.0/ci/lre/Jenkinsfile (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  properties([[$class: 'ThrottleJobProperty', categories: [], limitOneJobWithMatchingParams: false, maxConcurrentPerNode: 1,
     5      maxConcurrentTotal: 1, paramsToUseForLimit: '', throttleEnabled: true, throttleOption: 'project'],
     6      [$class: 'BuildDiscarderProperty',    strategy: [$class: 'LogRotator', numToKeepStr: '10']],
     7      pipelineTriggers([cron('0 11 * * *')]),])
     8  
     9  pipeline {
    10      options {
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16          docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              label 'internal'
    21          }
    22      }
    23  
    24      parameters {
    25          choice (name: 'LRE_CLUSTER',
    26                  description: 'Which LRE cluster to run the test against',
    27                  // 1st choice is the default value
    28                  // Currently, dev is the only option
    29                  choices: [ "dev" ])
    30          booleanParam (name: 'UPGRADE_VZ',
    31                  description: 'If true, upgrade Verrazzano to the most recent stable commit',
    32                  defaultValue: true)
    33          booleanParam (name: 'RUN_TESTS',
    34                  description: 'If true, run tests to verify the status of the cluster',
    35                  defaultValue: true)
    36      }
    37  
    38      environment {
    39          LRE_CLUSTER_ENV = "${params.LRE_CLUSTER}"
    40  
    41          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
    42          OCI_OS_BUCKET = "verrazzano-builds"
    43          OCI_OS_COMMIT_BUCKET = "verrazzano-builds-by-commit"
    44  
    45          OCI_CLI_TENANCY = credentials('oci-dev-tenancy')
    46          OCI_CLI_USER = credentials('oci-dev-user-ocid')
    47          OCI_CLI_FINGERPRINT = credentials('oci-dev-api-key-fingerprint')
    48          OCI_CLI_KEY_FILE = credentials('oci-dev-api-key-file')
    49          OCI_CLI_REGION = "us-ashburn-1"
    50  
    51          STABLE_COMMIT_OS_LOCATION = "master/last-stable-commit.txt"
    52          STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt"
    53          OPERATOR_YAML_LOCATION = "${WORKSPACE}/operator.yaml"
    54  
    55          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    56          DOCKER_REPO = 'ghcr.io'
    57          NETRC_FILE = credentials('netrc')
    58          GOPATH = '/home/opc/go'
    59          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    60          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
    61  
    62          DEV_LRE_SUSPECT_LIST = credentials('dev_lre_suspect_list')
    63          PIPELINE_OWNERS = credentials('lretests-owners')
    64          DEV_LRE_KUBECONFIG = credentials('dev-lre-kubeconfig')
    65          DEV_LRE_RANCHER_ADMIN_PSW = credentials('dev-lre-rancher-admin-password')
    66          KUBECONFIG = "${WORKSPACE}/.kube/config"
    67          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
    68          TEST_ENV = "LRE"
    69  
    70          // Environment variable for Verrazzano CLI executable
    71          VZ_COMMAND="${GO_REPO_PATH}/vz"
    72      }
    73  
    74      stages {
    75          stage('Clean workspace and checkout') {
    76              steps {
    77                  sh """
    78                      echo "${NODE_LABELS}"
    79                  """
    80  
    81                  script {
    82                      def scmInfo = checkout scm
    83                      env.GIT_BRANCH = scmInfo.GIT_BRANCH
    84                  }
    85                  sh """
    86                      cp -f "${NETRC_FILE}" $HOME/.netrc
    87                      chmod 600 $HOME/.netrc
    88                  """
    89  
    90                  script {
    91                      try {
    92                          sh """
    93                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
    94                          """
    95                      } catch(error) {
    96                          echo "docker login failed, retrying after sleep"
    97                          retry(4) {
    98                              sleep(30)
    99                              sh """
   100                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   101                              """
   102                          }
   103                      }
   104                  }
   105                  sh """
   106                      rm -rf ${GO_REPO_PATH}/verrazzano
   107                      mkdir -p ${GO_REPO_PATH}/verrazzano
   108                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   109                  """
   110                  sh """
   111                      echo "Downloading VZ CLI from object storage, lre is special only get latest one from master"
   112                      oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name master/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   113                      tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   114                      ${VZ_COMMAND} version
   115                  """
   116              }
   117          }
   118  
   119          stage('Set kubeconfig') {
   120              steps {
   121                  script {
   122                      sh """
   123                          mkdir -p ${WORKSPACE}/.kube
   124                          rm -rf ${KUBECONFIG}
   125  
   126                          RANCHER_TOKEN=\$(curl -s -k --noproxy rancher.lre.dev.v8odev01iad.oraclevcn.com -X POST \
   127                          -H 'Content-Type: application/json' \
   128                          "https://rancher.lre.dev.v8odev01iad.oraclevcn.com/v3-public/localProviders/local?action=login" \
   129                          -d '{"username":"admin", "password":"${DEV_LRE_RANCHER_ADMIN_PSW}"}' | jq -r ".token")
   130  
   131                          curl -s -k --noproxy rancher.lre.dev.v8odev01iad.oraclevcn.com -X POST \
   132                          -H "Authorization: Bearer \${RANCHER_TOKEN}" \
   133                          -H 'Content-Type: application/json' \
   134                          "https://rancher.lre.dev.v8odev01iad.oraclevcn.com/v3/clusters/local?action=generateKubeconfig" \
   135                          | jq -r ".config" > ${KUBECONFIG}
   136  
   137                          cat ${KUBECONFIG}
   138                      """
   139                  }
   140              }
   141          }
   142  
   143          stage ("Run kubectl commands") {
   144              steps {
   145                  script {
   146                      sh """
   147                          kubectl get nodes
   148                          kubectl get pods -A
   149                          kubectl get vz -A
   150                      """
   151                  }
   152              }
   153          }
   154  
   155          stage("Create OCI config file") {
   156              when {
   157                  expression {
   158                      return params.UPGRADE_VZ
   159                  }
   160              }
   161              steps {
   162                  script{
   163                      sh """
   164                          mkdir -p ~/.oci
   165                          cp ${OCI_CLI_KEY_FILE} ~/.oci
   166                          rm -rf ~/.oci/config
   167                          {
   168                            echo '[DEFAULT]'
   169                            echo 'user=${OCI_CLI_USER}'
   170                            echo 'fingerprint=${OCI_CLI_FINGERPRINT}'
   171                            echo 'tenancy=${OCI_CLI_TENANCY}'
   172                            echo 'region=${OCI_CLI_REGION}'
   173                            echo 'key_file=~/.oci/alm.pem'
   174                          } >> ~/.oci/config
   175                          oci setup repair-file-permissions --file /home/opc/.oci/config
   176                      """
   177                  }
   178              }
   179          }
   180  
   181          stage('Upgrade to stable commit') {
   182              when {
   183                  expression {
   184                      return params.UPGRADE_VZ
   185                  }
   186              }
   187              steps {
   188                  script {
   189                      // Get the last stable commit ID that passed the triggered tests
   190                      sh """
   191                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${STABLE_COMMIT_OS_LOCATION} --file ${STABLE_COMMIT_LOCATION}
   192                      """
   193                      def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}"
   194                      GIT_COMMIT_TO_USE = stableCommitProps['git-commit']
   195                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 ${GIT_COMMIT_TO_USE}").trim()
   196                      echo "Last stable short commit: ${SHORT_COMMIT_HASH}"
   197  
   198                      // Get the running version of Verrazzano
   199                      RUNNING_VERSION = sh(returnStdout: true, script: "kubectl get verrazzano -A -o jsonpath='{.items[0].status.version}'").trim()
   200                      echo "Current version of Verrazzano: ${RUNNING_VERSION}"
   201  
   202                      // Get the version we are upgrading to
   203                      VERRAZZANO_UPGRADE_VERSION = sh(returnStdout: true, script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/master/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file - | jq -r '.version'").trim()
   204                      echo "Version to upgrade to: ${VERRAZZANO_UPGRADE_VERSION}"
   205  
   206                      // Only do an upgrade if the running version and version to upgrade are different.
   207                      if (VERRAZZANO_UPGRADE_VERSION != RUNNING_VERSION)  {
   208                          sh """
   209                              echo "Downloading operator.yaml from object storage"
   210                              oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/master/${SHORT_COMMIT_HASH}/operator.yaml --file ${OPERATOR_YAML_LOCATION}
   211                          """
   212  
   213                          downloadCLI()
   214  
   215                          sh """
   216                              echo "Upgrading the Verrazzano installation to version" ${VERRAZZANO_UPGRADE_VERSION}
   217                              ${GO_REPO_PATH}/vz upgrade --version ${VERRAZZANO_UPGRADE_VERSION} --manifests ${OPERATOR_YAML_LOCATION} --timeout 45m
   218                          """
   219                      } else {
   220                          echo "No upgrade is needed.  Verrazzano is already at the expected version."
   221                      }
   222                  }
   223              }
   224              post {
   225                  failure {
   226                      dumpK8sCluster('lre-upgrade-failure-cluster-snapshot')
   227                  }
   228              }
   229          }
   230  
   231          stage('Verify Install') {
   232              when {
   233                 expression {
   234                     return params.RUN_TESTS
   235                 }
   236              }
   237              steps {
   238                  runGinkgoRandomize('verify-install')
   239              }
   240              post {
   241                  always {
   242                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   243                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   244                  }
   245                  failure {
   246                      dumpK8sCluster('lre-verify-install-failure-cluster-snapshot')
   247                  }
   248              }
   249          }
   250  
   251          stage('Run Acceptance Tests Infra') {
   252              when {
   253                 expression {
   254                     return params.RUN_TESTS
   255                 }
   256              }
   257              parallel {
   258                  stage('verify-scripts') {
   259                      steps {
   260                          runGinkgoRandomize('scripts', "${KUBECONFIG}")
   261                      }
   262                  }
   263                  stage('verify-infra restapi') {
   264                      steps {
   265                          runGinkgoRandomize('verify-infra/restapi')
   266                      }
   267                  }
   268                  stage('verify-infra oam') {
   269                      steps {
   270                          runGinkgoRandomize('verify-infra/oam')
   271                      }
   272                  }
   273  
   274                  stage('verify-infra vmi') {
   275                      steps {
   276                          runGinkgoRandomize('verify-infra/vmi')
   277                      }
   278                  }
   279  
   280                  stage('system component metrics') {
   281                      steps {
   282                          runGinkgo('metrics/syscomponents')
   283                      }
   284                  }
   285              }
   286              post {
   287                  always {
   288                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   289                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   290                  }
   291                  failure {
   292                      dumpK8sCluster('lre-verify-infra-failure-cluster-snapshot')
   293                  }
   294              }
   295          }
   296  
   297          stage('examples helidon') {
   298              when {
   299                 expression {
   300                     return params.RUN_TESTS
   301                 }
   302              }
   303              steps {
   304                  script {
   305                      runGinkgoAppTest('examples/helidon', "hello-helidon", "${TEST_DUMP_ROOT}/examples-helidon", 'true', 'true')
   306                  }
   307              }
   308              post {
   309                  always {
   310                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   311                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   312                  }
   313                  failure {
   314                      dumpK8sCluster('lre-hello-helidon-failure-cluster-snapshot')
   315                  }
   316              }
   317          }
   318  
   319      }
   320      post {
   321          failure {
   322              script {
   323                  if (env.GIT_BRANCH == "master") {
   324                      slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${DEV_LRE_SUSPECT_LIST} ${PIPELINE_OWNERS}\n" )
   325                  }
   326              }
   327          }
   328      }
   329  }
   330  
   331  def downloadCLI() {
   332      script {
   333          sh """
   334              echo "Downloading vz CLI from object storage"
   335              oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/master/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   336              tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   337              ${GO_REPO_PATH}/vz version
   338          """
   339      }
   340  }
   341  
   342  def runGinkgoRandomize(testSuitePath, kubeconfig = '') {
   343      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   344          sh """
   345              if [ ! -z "${kubeConfig}" ]; then
   346                  export KUBECONFIG="${kubeConfig}"
   347              fi
   348              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   349              ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/...
   350          """
   351      }
   352  }
   353  
   354  def runGinkgo(testSuitePath) {
   355      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   356          sh """
   357              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   358              ginkgo -v --keep-going --no-color ${testSuitePath}/...
   359          """
   360      }
   361  }
   362  
   363  def runGinkgoAppTest(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false') {
   364      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   365          sh """
   366              if [ ! -z "${dumpDir}" ]; then
   367                  export DUMP_DIRECTORY=${dumpDir}
   368              fi
   369              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   370              ginkgo -v --keep-going --no-color ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace}
   371          """
   372      }
   373  }
   374  
   375  def dumpK8sCluster(dumpDirectory) {
   376      sh """
   377          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   378      """
   379  }