github.com/verrazzano/verrazzano@v1.7.1/ci/lre/Jenkinsfile (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  properties([[$class: 'ThrottleJobProperty', categories: [], limitOneJobWithMatchingParams: false, maxConcurrentPerNode: 1,
     5      maxConcurrentTotal: 1, paramsToUseForLimit: '', throttleEnabled: true, throttleOption: 'project'],
     6      [$class: 'BuildDiscarderProperty',    strategy: [$class: 'LogRotator', numToKeepStr: '10']],
     7      pipelineTriggers([cron('0 11 * * *')]),])
     8  
     9  pipeline {
    10      options {
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16          docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              label 'internal'
    21          }
    22      }
    23  
    24      parameters {
    25          choice (name: 'LRE_CLUSTER',
    26                  description: 'Which LRE cluster to run the test against',
    27                  // 1st choice is the default value
    28                  // Currently, dev is the only option
    29                  choices: [ "dev" ])
    30          booleanParam (name: 'UPGRADE_VZ',
    31                  description: 'If true, upgrade Verrazzano to the most recent stable commit',
    32                  defaultValue: true)
    33          booleanParam (name: 'RUN_TESTS',
    34                  description: 'If true, run tests to verify the status of the cluster',
    35                  defaultValue: true)
    36      }
    37  
    38      environment {
    39          LRE_CLUSTER_ENV = "${params.LRE_CLUSTER}"
    40  
    41          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
    42  
    43          OCI_CLI_TENANCY = credentials('oci-dev-tenancy')
    44          OCI_CLI_USER = credentials('oci-dev-user-ocid')
    45          OCI_CLI_FINGERPRINT = credentials('oci-dev-api-key-fingerprint')
    46          OCI_CLI_KEY_FILE = credentials('oci-dev-api-key-file')
    47          OCI_CLI_REGION = "us-ashburn-1"
    48  
    49          STABLE_COMMIT_OS_LOCATION = "master/last-stable-commit.txt"
    50          STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt"
    51          OPERATOR_YAML_LOCATION = "${WORKSPACE}/operator.yaml"
    52  
    53          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    54          DOCKER_REPO = 'ghcr.io'
    55          NETRC_FILE = credentials('netrc')
    56          GOPATH = '/home/opc/go'
    57          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    58          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
    59  
    60          DEV_LRE_SUSPECT_LIST = credentials('dev_lre_suspect_list')
    61          PIPELINE_OWNERS = credentials('lretests-owners')
    62          DEV_LRE_KUBECONFIG = credentials('dev-lre-kubeconfig')
    63          DEV_LRE_RANCHER_ADMIN_PSW = credentials('dev-lre-rancher-admin-password')
    64          KUBECONFIG = "${WORKSPACE}/.kube/config"
    65          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
    66          TEST_ENV = "LRE"
    67  
    68          // Environment variable for Verrazzano CLI executable
    69          VZ_COMMAND="${GO_REPO_PATH}/vz"
    70      }
    71  
    72      stages {
    73          stage('Clean workspace and checkout') {
    74              steps {
    75                  sh """
    76                      echo "${NODE_LABELS}"
    77                  """
    78  
    79                  script {
    80                      def scmInfo = checkout scm
    81                      env.GIT_BRANCH = scmInfo.GIT_BRANCH
    82                  }
    83                  sh """
    84                      cp -f "${NETRC_FILE}" $HOME/.netrc
    85                      chmod 600 $HOME/.netrc
    86                  """
    87  
    88                  script {
    89                      try {
    90                          sh """
    91                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
    92                          """
    93                      } catch(error) {
    94                          echo "docker login failed, retrying after sleep"
    95                          retry(4) {
    96                              sleep(30)
    97                              sh """
    98                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
    99                              """
   100                          }
   101                      }
   102                  }
   103                  sh """
   104                      rm -rf ${GO_REPO_PATH}/verrazzano
   105                      mkdir -p ${GO_REPO_PATH}/verrazzano
   106                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   107                  """
   108                  sh """
   109                      echo "Downloading VZ CLI from object storage, lre is special only get latest one from master"
   110                      oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name master/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   111                      tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   112                      ${VZ_COMMAND} version
   113                  """
   114              }
   115          }
   116  
   117          stage('Set kubeconfig') {
   118              steps {
   119                  script {
   120                      sh """
   121                          mkdir -p ${WORKSPACE}/.kube
   122                          rm -rf ${KUBECONFIG}
   123  
   124                          RANCHER_TOKEN=\$(curl -s -k --noproxy rancher.lre.dev.v8odev01iad.oraclevcn.com -X POST \
   125                          -H 'Content-Type: application/json' \
   126                          "https://rancher.lre.dev.v8odev01iad.oraclevcn.com/v3-public/localProviders/local?action=login" \
   127                          -d '{"username":"admin", "password":"${DEV_LRE_RANCHER_ADMIN_PSW}"}' | jq -r ".token")
   128  
   129                          curl -s -k --noproxy rancher.lre.dev.v8odev01iad.oraclevcn.com -X POST \
   130                          -H "Authorization: Bearer \${RANCHER_TOKEN}" \
   131                          -H 'Content-Type: application/json' \
   132                          "https://rancher.lre.dev.v8odev01iad.oraclevcn.com/v3/clusters/local?action=generateKubeconfig" \
   133                          | jq -r ".config" > ${KUBECONFIG}
   134  
   135                          cat ${KUBECONFIG}
   136                      """
   137                  }
   138              }
   139          }
   140  
   141          stage ("Run kubectl commands") {
   142              steps {
   143                  script {
   144                      sh """
   145                          kubectl get nodes
   146                          kubectl get pods -A
   147                          kubectl get vz -A
   148                      """
   149                  }
   150              }
   151          }
   152  
   153          stage("Create OCI config file") {
   154              when {
   155                  expression {
   156                      return params.UPGRADE_VZ
   157                  }
   158              }
   159              steps {
   160                  script{
   161                      sh """
   162                          mkdir -p ~/.oci
   163                          cp ${OCI_CLI_KEY_FILE} ~/.oci
   164                          rm -rf ~/.oci/config
   165                          {
   166                            echo '[DEFAULT]'
   167                            echo 'user=${OCI_CLI_USER}'
   168                            echo 'fingerprint=${OCI_CLI_FINGERPRINT}'
   169                            echo 'tenancy=${OCI_CLI_TENANCY}'
   170                            echo 'region=${OCI_CLI_REGION}'
   171                            echo 'key_file=~/.oci/alm.pem'
   172                          } >> ~/.oci/config
   173                          oci setup repair-file-permissions --file /home/opc/.oci/config
   174                      """
   175                  }
   176              }
   177          }
   178  
   179          stage('Upgrade to stable commit') {
   180              when {
   181                  expression {
   182                      return params.UPGRADE_VZ
   183                  }
   184              }
   185              steps {
   186                  script {
   187                      // Get the last stable commit ID that passed the triggered tests
   188                      sh """
   189                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${STABLE_COMMIT_OS_LOCATION} --file ${STABLE_COMMIT_LOCATION}
   190                      """
   191                      def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}"
   192                      GIT_COMMIT_TO_USE = stableCommitProps['git-commit']
   193                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 ${GIT_COMMIT_TO_USE}").trim()
   194                      echo "Last stable short commit: ${SHORT_COMMIT_HASH}"
   195  
   196                      // Get the running version of Verrazzano
   197                      RUNNING_VERSION = sh(returnStdout: true, script: "kubectl get verrazzano -A -o jsonpath='{.items[0].status.version}'").trim()
   198                      echo "Current version of Verrazzano: ${RUNNING_VERSION}"
   199  
   200                      // Get the version we are upgrading to
   201                      VERRAZZANO_UPGRADE_VERSION = sh(returnStdout: true, script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/master/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file - | jq -r '.version'").trim()
   202                      echo "Version to upgrade to: ${VERRAZZANO_UPGRADE_VERSION}"
   203  
   204                      // Only do an upgrade if the running version and version to upgrade are different.
   205                      if (VERRAZZANO_UPGRADE_VERSION != RUNNING_VERSION)  {
   206                          sh """
   207                              echo "Downloading operator.yaml from object storage"
   208                              oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/master/${SHORT_COMMIT_HASH}/operator.yaml --file ${OPERATOR_YAML_LOCATION}
   209                          """
   210  
   211                          downloadCLI()
   212  
   213                          sh """
   214                              echo "Upgrading the Verrazzano installation to version" ${VERRAZZANO_UPGRADE_VERSION}
   215                              ${GO_REPO_PATH}/vz upgrade --version ${VERRAZZANO_UPGRADE_VERSION} --manifests ${OPERATOR_YAML_LOCATION} --timeout 45m
   216                          """
   217                      } else {
   218                          echo "No upgrade is needed.  Verrazzano is already at the expected version."
   219                      }
   220                  }
   221              }
   222              post {
   223                  failure {
   224                      dumpK8sCluster('lre-upgrade-failure-cluster-snapshot')
   225                  }
   226              }
   227          }
   228  
   229          stage('Verify Install') {
   230              when {
   231                 expression {
   232                     return params.RUN_TESTS
   233                 }
   234              }
   235              steps {
   236                  runGinkgoRandomize('verify-install')
   237              }
   238              post {
   239                  always {
   240                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   241                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   242                  }
   243                  failure {
   244                      dumpK8sCluster('lre-verify-install-failure-cluster-snapshot')
   245                  }
   246              }
   247          }
   248  
   249          stage('Run Acceptance Tests Infra') {
   250              when {
   251                 expression {
   252                     return params.RUN_TESTS
   253                 }
   254              }
   255              parallel {
   256                  stage('verify-scripts') {
   257                      steps {
   258                          runGinkgoRandomize('scripts', "${KUBECONFIG}")
   259                      }
   260                  }
   261                  stage('verify-infra restapi') {
   262                      steps {
   263                          runGinkgoRandomize('verify-infra/restapi')
   264                      }
   265                  }
   266                  stage('verify-infra oam') {
   267                      steps {
   268                          runGinkgoRandomize('verify-infra/oam')
   269                      }
   270                  }
   271  
   272                  stage('verify-infra vmi') {
   273                      steps {
   274                          runGinkgoRandomize('verify-infra/vmi')
   275                      }
   276                  }
   277  
   278                  stage('system component metrics') {
   279                      steps {
   280                          runGinkgo('metrics/syscomponents')
   281                      }
   282                  }
   283              }
   284              post {
   285                  always {
   286                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   287                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   288                  }
   289                  failure {
   290                      dumpK8sCluster('lre-verify-infra-failure-cluster-snapshot')
   291                  }
   292              }
   293          }
   294  
   295          stage('examples helidon') {
   296              when {
   297                 expression {
   298                     return params.RUN_TESTS
   299                 }
   300              }
   301              steps {
   302                  script {
   303                      runGinkgoAppTest('examples/helidon', "hello-helidon", "${TEST_DUMP_ROOT}/examples-helidon", 'true', 'true')
   304                  }
   305              }
   306              post {
   307                  always {
   308                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   309                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   310                  }
   311                  failure {
   312                      dumpK8sCluster('lre-hello-helidon-failure-cluster-snapshot')
   313                  }
   314              }
   315          }
   316  
   317      }
   318      post {
   319          failure {
   320              script {
   321                  if (env.GIT_BRANCH == "master") {
   322                      slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${DEV_LRE_SUSPECT_LIST} ${PIPELINE_OWNERS}\n" )
   323                  }
   324              }
   325          }
   326      }
   327  }
   328  
   329  def downloadCLI() {
   330      script {
   331          sh """
   332              echo "Downloading vz CLI from object storage"
   333              oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/master/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   334              tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   335              ${GO_REPO_PATH}/vz version
   336          """
   337      }
   338  }
   339  
   340  def runGinkgoRandomize(testSuitePath, kubeconfig = '') {
   341      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   342          sh """
   343              if [ ! -z "${kubeConfig}" ]; then
   344                  export KUBECONFIG="${kubeConfig}"
   345              fi
   346              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   347              ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/...
   348          """
   349      }
   350  }
   351  
   352  def runGinkgo(testSuitePath) {
   353      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   354          sh """
   355              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   356              ginkgo -v --keep-going --no-color ${testSuitePath}/...
   357          """
   358      }
   359  }
   360  
   361  def runGinkgoAppTest(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false') {
   362      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   363          sh """
   364              if [ ! -z "${dumpDir}" ]; then
   365                  export DUMP_DIRECTORY=${dumpDir}
   366              fi
   367              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   368              ginkgo -v --keep-going --no-color ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace}
   369          """
   370      }
   371  }
   372  
   373  def dumpK8sCluster(dumpDirectory) {
   374      sh """
   375          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   376      """
   377  }