github.com/verrazzano/verrazzano@v1.7.1/ci/generic/Jenkinsfile

// Copyright (c) 2023, 2024, Oracle and/or its affiliates.
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.

def DOCKER_IMAGE_TAG
def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false

targetsKey = "targets"
displayNameKey = "displayname"
unknownTargetName = "UNKNOWN"
testGroupsKey = "testGroups"

pipeline {
    options {
        timeout(time: 1, unit: 'HOURS')
        skipDefaultCheckout true
        timestamps ()
    }

    agent {
       docker {
            image "${RUNNER_DOCKER_IMAGE}"
            args "${RUNNER_DOCKER_ARGS}"
            registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
            registryCredentialsId 'ocir-pull-and-push-account'
            label "${agentLabel}"
        }
    }

    parameters {
        choice (name: 'KUBERNETES_CLUSTER_VERSION',
                description: 'Kubernetes Version for KinD Cluster',
                // 1st choice is the default value
                choices: [ "1.27", "1.26","1.25", "1.24" ])
        string (name: 'GIT_COMMIT_TO_USE',
                        defaultValue: 'NONE',
                        description: 'This is the full git commit hash from the source build to be used for all jobs',
                        trim: true)
        string (name: 'PIPELINE_DESCRIPTOR',
                        defaultValue: 'ci/generic/kind-acceptance-tests.yaml',
                        description: 'Pipeline descriptor file location, relative to base of Verrazzano repo',
                        trim: true)
        string (name: 'VERRAZZANO_OPERATOR_IMAGE',
                        defaultValue: 'NONE',
                        description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
                        trim: true)
        choice (name: 'WILDCARD_DNS_DOMAIN',
                description: 'This is the wildcard DNS domain',
                // 1st choice is the default value
                choices: [ "nip.io", "sslip.io"])
        choice (name: 'CRD_API_VERSION',
                description: 'This is the API crd version.',
                // 1st choice is the default value
                choices: [ "v1beta1", "v1alpha1"])
        booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
        booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
        string (name: 'CONSOLE_REPO_BRANCH',
                defaultValue: '',
                description: 'The branch to check out after cloning the console repository.',
                trim: true)
        booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
        string (name: 'TAGGED_TESTS',
                defaultValue: '',
                description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
                trim: true)
        string (name: 'INCLUDED_TESTS',
                defaultValue: '.*',
                description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
                trim: true)
        string (name: 'EXCLUDED_TESTS',
                defaultValue: '_excluded_test',
                description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
                trim: true)
        booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    }

    environment {
        DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
        DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
        GOPATH = '/home/opc/go'
        GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
        DOCKER_CREDS = credentials('github-packages-credentials-rw')
        DOCKER_EMAIL = credentials('github-packages-email')
        DOCKER_REPO = 'ghcr.io'
        DOCKER_NAMESPACE = 'verrazzano'

        NETRC_FILE = credentials('netrc')
        CLUSTER_NAME = 'verrazzano'
        POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
        TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
        KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
        VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
        OCR_CREDS = credentials('ocr-pull-and-push-account')
        OCR_REPO = 'container-registry.oracle.com'
        IMAGE_PULL_SECRET = 'verrazzano-container-registry'
        INSTALL_PROFILE = "dev"
        VZ_ENVIRONMENT_NAME = "default"
        TEST_ROOT = "${WORKSPACE}/tests/e2e"
        TEST_SCRIPTS_DIR = "${TEST_ROOT}/config/scripts"
        INSTALL_CONFIG_FILE_KIND = "${TEST_SCRIPTS_DIR}/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
        VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"

        WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
        DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test

        // Environment variables required to capture cluster snapshot and bug report on test failure
        DUMP_KUBECONFIG="${KUBECONFIG}"
        DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
        DUMP_ROOT_DIRECTORY="${WORKSPACE}/test-cluster-snapshots"
        CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"

        // Environment variable for Verrazzano CLI executable
        VZ_COMMAND="${GO_REPO_PATH}/vz"

        POST_INSTALL_DUMP="true"

        VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
        VERRAZZANO_INSTALL_LOG="verrazzano-install.log"

        // used for console artifact capture on failure
        JENKINS_READ = credentials('jenkins-auditor')
        OCI_CLI_AUTH="instance_principal"
        OCI_OS_NAMESPACE = credentials('oci-os-namespace')
        OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"

        // used to emit metrics
        PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
        TEST_ENV_LABEL = "kind"
        TEST_ENV = "KIND"
        K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"

        // used to generate Ginkgo test reports
        TEST_REPORT = "test-report.xml"
        TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
    }

    stages {
        stage('Clean workspace and checkout') {
            steps {
                pipelineSetup()
            }
        }

        stage('Run Acceptance Tests') {
            environment {
                KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
                OCI_CLI_AUTH="instance_principal"
                OCI_OS_NAMESPACE = credentials('oci-os-namespace')
                OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
                PIPELINE_CONFIG=getPipelineDescriptorPath()
            }
            steps {
                echo "Executing pipeline configuration ${env.PIPELINE_CONFIG}"
                runDynamicStages(env.PIPELINE_CONFIG)
            }
            post {
                always {
                    archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/*${TEST_REPORT}", allowEmptyArchive: true
                    junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
                }
                failure {
                    script {
                        if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
                            dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
                        }
                    }
                }
                success {
                    script {
                        if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
                            dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
                        }
                    }
                }
            }
        }
    }
    post {
        always {
            archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*cluster-snapshot*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/*${TEST_REPORT}", allowEmptyArchive: true
            junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true

            script {
                runMakeCommand("cleanup")
            }
        }
        failure {
            postFailureProcessing()
        }
        cleanup {
            deleteDir()
        }
    }
}

def getPipelineDescriptorPath() {
    return "${WORKSPACE}/${params.PIPELINE_DESCRIPTOR}"
}

def getPipelineName() {
    data = readYaml file: getPipelineDescriptorPath()
    return data.metadata.name
}

def runDynamicStages(pipelineDescriptor) {
    script {
        data = readYaml file: pipelineDescriptor
        println data
        print "Name: " + data.metadata.name
        print "Agent label: " + data.metadata.agent["label"]

        testGroups = [:]
        if (data.containsKey(testGroupsKey)) {
            testGroups = processTestGroups(data[testGroupsKey])
        }
        print "Test groups: " + testGroups

        print "Stages: " + data.stages
        stages = data.stages
        for (stageDescriptor in stages) {
            stageName = stageDescriptor.name
            print "Stage: " + stageName
            targetsMap = [:]
            if (stageDescriptor.containsKey(targetsKey)) {
                targetsMap.putAll(processTargets(stageDescriptor[targetsKey]))
            }
            if (stageDescriptor.containsKey(testGroupsKey)) {
                groupNames = stageDescriptor[testGroupsKey]
                targetsMap.putAll(addTestGroups(testGroups, groupNames))
            }
            print "Stage Map to use: " + targetsMap
            stage(stageName) {
                try {
                    echo "Executing stage ${stageName}, targets " + targetsMap.keySet()
                    parallel targetsMap
                } finally {
                    echo "Archiving artifacts for stage ${stageName}"
                    archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/*${TEST_REPORT},*/Screenshot*.png,**/ConsoleLog*.log,**/*cluster-snapshot*/**', allowEmptyArchive: true
                    junit testResults: '**/*test-result.xml', allowEmptyResults: true
                }
            }
        }
    }
}

// Returns an map of all targets keyed by display name
def processTargets(targets) {
    targetMap = [:]
    for (target in targets) {
        targetName = target.name
        if (target.containsKey(displayNameKey)) {
            targetName = target[displayNameKey]
        }
        if (targetName.length() == 0) {
            targetName = unknownTargetName
        }
        targetMap.putAll(buildTargetClosure(targetName, target.target))
    }
    return targetMap
}

// Returns an aggregate Map of all targets associated with the list of test groups provided
def addTestGroups(testGroups, groupNames) {
    targetsMap = [:]
    for (groupName in groupNames) {
        if (!testGroups.containsKey(groupName)) {
            println("Group ${group} not found in groups map!")
            continue
        }
        targets = testGroups[groupName]
        targetsMap.putAll(processTargets(targets))
    }
    return targetsMap
}

def processTestGroups(testGroups) {
    groupsMap = [:]
    for (group in testGroups) {
        groupsMap.put(group.name, group.targets)
    }
    return groupsMap
}

def buildTargetClosure(targetName, targetToInvoke) {
    echo "Building closure for ${targetToInvoke}"
    return [ (targetName) : {
            echo "Running target: ${targetToInvoke}"
            runMakeCommand(targetToInvoke)
        }
    ]
}

def runTestTarget(testSuitePath, runParallel = "true", randomize = "true") {
    return script {
        sh """
           export TEST_SUITES="${testSuitePath}/..."
           export RANDOMIZE_TESTS=${randomize}
           export RUN_PARALLEL=${runParallel}
           cd ${GO_REPO_PATH}/verrazzano/ci/generic
           make test
        """
    }
}

def runMakeCommand(makeTarget) {
    sh """
       cd ${GO_REPO_PATH}/verrazzano/ci/generic
       make ${makeTarget}
    """
}

def pipelineSetup() {
    sh """
        echo "${NODE_LABELS}"
    """

    script {
        EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
        if (params.GIT_COMMIT_TO_USE == "NONE") {
            echo "Specific GIT commit was not specified, use current head"
            def scmInfo = checkout scm
            env.GIT_COMMIT = scmInfo.GIT_COMMIT
            env.GIT_BRANCH = scmInfo.GIT_BRANCH
        } else {
            echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
            def scmInfo = checkout([
                $class: 'GitSCM',
                branches: [[name: params.GIT_COMMIT_TO_USE]],
                doGenerateSubmoduleConfigurations: false,
                extensions: [],
                submoduleCfg: [],
                userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
            env.GIT_COMMIT = scmInfo.GIT_COMMIT
            env.GIT_BRANCH = scmInfo.GIT_BRANCH
            // If the commit we were handed is not what the SCM says we are using, fail
            if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
                echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
                exit 1
            }
        }
        echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
    }

    sh """
        cp -f "${NETRC_FILE}" $HOME/.netrc
        chmod 600 $HOME/.netrc
    """

    script {
        try {
        sh """
            echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
        """
        } catch(error) {
            echo "docker login failed, retrying after sleep"
            retry(4) {
                sleep(30)
                sh """
                    echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
                """
            }
        }
    }
    sh """
        rm -rf ${GO_REPO_PATH}/verrazzano
        mkdir -p ${GO_REPO_PATH}/verrazzano
        tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
    """

    script {
        def props = readProperties file: '.verrazzano-development-version'
        VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
        TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
        SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
        DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
        // update the description with some meaningful info
        setDisplayName()
        currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
    }
}

def postFailureProcessing() {
    sh """
        curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
    """
    archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
    sh """
        curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
        oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
        rm archive.zip
    """
    script {
        if (env.BRANCH_NAME == "master" || env.BRANCH_NAME ==~ "release-.*" || env.BRANCH_NAME ==~ "mark/*") {
            slackSend ( message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}" )
        }
    }
}

def getEffectiveDumpOnSuccess() {
    def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
    if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
        effectiveValue = true
        echo "Forcing dump on success based on global override setting"
    }
    return effectiveValue
}

def setDisplayName() {
    echo "Start setDisplayName"
    def causes = currentBuild.getBuildCauses()
    echo "causes: " + causes.toString()
    for (cause in causes) {
        def causeString = cause.toString()
        echo "current cause: " + causeString
        if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
             echo "This job was caused by " + causeString
             if (causeString.contains("verrazzano-periodic-triggered-tests")) {
                 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
             } else if (causeString.contains("verrazzano-flaky-tests")) {
                 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
             }
         }
    }
    echo "End setDisplayName"
}

def dumpK8sCluster(dumpDirectory) {
    sh """
        ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
    """
}