github.com/verrazzano/verrazzano@v1.7.1/ci/scripts/get_branch_scan_results.sh (about) 1 #!/usr/bin/env bash 2 # 3 # Copyright (c) 2021, 2024, Oracle and/or its affiliates. 4 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 5 # 6 7 # NOTE: This script assumes that: 8 # 9 # 1) "docker login" has been done for the image registry 10 # 2) OCI credentials have been configured to allow the OCI CLI to fetch scan results from OCIR 11 # 3) "gh auth login" has been done to allow the github CLI to list releases and fetch release artifacts 12 # 13 SCRIPT_DIR=$(cd $(dirname "$0"); pwd -P) 14 RELEASE_SCRIPT_DIR=${SCRIPT_DIR}/../../release/scripts 15 16 if [ -z "$JENKINS_URL" ] || [ -z "$WORKSPACE" ] || [ -z "$OCI_OS_NAMESPACE" ] || [ -z "$OCI_OS_BUCKET" ] || [ -z "$OCI_SCAN_BUCKET" ] || [ -z "$CLEAN_BRANCH_NAME" ]; then 17 echo "This script must only be called from Jenkins and requires a number of environment variables are set" 18 exit 1 19 fi 20 21 BASE_OBJ_PATH="daily-scan/${CLEAN_BRANCH_NAME}" 22 SCAN_DATETIME="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" 23 JOB_OBJ_PATH="${BASE_OBJ_PATH}/${SCAN_DATETIME}-${BUILD_NUMBER}" 24 25 # Hack to get the generated BOM from a release by pulling down the operator.yaml from the release artifacts 26 # and copying the BOM from the platform operator image 27 function get_bom_from_release() { 28 local releaseTag=$1 29 local outputFile=$2 30 local tmpDir=$(mktemp -d) 31 32 # Download the operator.yaml for the release and get the platform-operator image and tag 33 local operator_yaml=$(derive_platform_operator "$releaseTag") 34 gh release download ${releaseTag} -p '${operator_yaml}' -D ${tmpDir} 35 local image=$(grep "verrazzano-platform-operator:" ${tmpDir}/${operator_yaml} | grep "image:" -m 1 | xargs | cut -d' ' -f 2) 36 37 # Create a container from the image and copy the BOM from the container 38 local containerId=$(docker create ${image}) 39 docker cp ${containerId}:/verrazzano/platform-operator/verrazzano-bom.json ${outputFile} 40 docker rm ${containerId} 41 42 rm -fr ${tmpDir} 43 } 44 45 # Publish the results to object storage 46 function publish_results() { 47 local resultName=$1 48 local bomFile=$2 49 local resultsDir=$3 50 51 zip -r ${WORKSPACE}/${resultName}-details.zip ${resultsDir} 52 53 # Push latest 54 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${BASE_OBJ_PATH}/${resultName}/verrazzano-bom.json --file ${bomFile} 55 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${BASE_OBJ_PATH}/${resultName}/consolidated-report.out --file ${resultsDir}/consolidated-report.out 56 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${BASE_OBJ_PATH}/${resultName}/consolidated.csv --file ${resultsDir}/consolidated.csv 57 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${BASE_OBJ_PATH}/${resultName}/consolidated-upload.json --file ${resultsDir}/consolidated-upload.json 58 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${BASE_OBJ_PATH}/${resultName}/details.zip --file ${WORKSPACE}/${resultName}-details.zip 59 60 # Push to job specific location 61 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${JOB_OBJ_PATH}/${resultName}/verrazzano-bom.json --file ${bomFile} 62 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${JOB_OBJ_PATH}/${resultName}/consolidated-report.out --file ${resultsDir}/consolidated-report.out 63 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${JOB_OBJ_PATH}/${resultName}/consolidated.csv --file ${resultsDir}/consolidated.csv 64 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${JOB_OBJ_PATH}/${resultName}/consolidated-upload.json --file ${resultsDir}/consolidated-upload.json 65 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_SCAN_BUCKET} --name ${JOB_OBJ_PATH}/${resultName}/details.zip --file ${WORKSPACE}/${resultName}-details.zip 66 } 67 68 # Read the value for a given key from effective.config.json 69 function derive_platform_operator() { 70 local release_tag="$1" 71 72 # Remove prefix v from version 73 local version_num=${release_tag:1} 74 75 # Verrazzano distribution from 1.4.0 release replaces operator.yaml with verrazzano-platform-operator.yaml in release assets 76 local version_14=1.4.0 77 local operator_yaml=$(echo ${VERSION_NUM} ${VERSION_14} | awk '{if ($1 < $2) print "operator.yaml"; else print "verrazzano-platform-operator.yaml"}') 78 echo $operator_yaml 79 return 0 80 } 81 82 BOM_DIR=${WORKSPACE}/boms 83 mkdir -p ${BOM_DIR} 84 SCAN_RESULTS_BASE_DIR=${WORKSPACE}/scan-results 85 export SCAN_RESULTS_DIR=${SCAN_RESULTS_BASE_DIR}/latest 86 mkdir -p ${SCAN_RESULTS_DIR} 87 88 if [[ -z "${SCANNER_PATH}" ]]; then 89 echo "Environment variable SCANNER_PATH is not set" 90 else 91 export PATH="${SCANNER_PATH}:${PATH}" 92 fi 93 94 # The script uses Github CLI and OCI CLI, check whether the CLIs are in PATH 95 command -v gh >/dev/null 2>&1 || { 96 echo "Github CLI is not in PATH" 97 exit 1 98 } 99 100 command -v oci >/dev/null 2>&1 || { 101 echo "OCI CLI is not in PATH" 102 exit 1 103 } 104 105 # Where the results are kept for the branch depend on what kind of branch it is and where the updated bom is stored: 106 # master, release-* branches are regularly updated using the periodic pipelines only 107 # 108 # The BOM for the latest results from the NORMAL workflows is here (master, release-*, special runs of branches): 109 # ${CLEAN_BRANCH_NAME}-last-clean-periodic-test/last-ocir-pushed-verrazzano-bom.json 110 # 111 # It is possible that someone ran a job which needed to specify that the tip of master or release-* push images to 112 # OCIR. This does NOT happen normally, the only situation where this is done from a pipeline is when performing a 113 # release that required a BUILD to be done (ie: when releasing something that was NOT pre-baked for some reason). 114 # In these cases, the BOM is stored here: 115 # 116 # ${CLEAN_BRANCH_NAME}-last-snapshot/last-ocir-pushed-verrazzano-bom.json 117 # 118 # all other branches only will be pushed if explicitly set as a parameter. In these cases, the BOM is stored here: 119 # 120 # ${CLEAN_BRANCH_NAME}/last-ocir-pushed-verrazzano-bom.json 121 122 # Get the last pushed BOMs for the branch 123 echo "Attempting to fetch BOM from object storage for branch: ${CLEAN_BRANCH_NAME}" 124 mkdir -p ${BOM_DIR}/${CLEAN_BRANCH_NAME}-last-clean-periodic-test 125 mkdir -p ${BOM_DIR}/${CLEAN_BRANCH_NAME}-last-snapshot 126 mkdir -p ${BOM_DIR}/${CLEAN_BRANCH_NAME} 127 export SCAN_BOM_PERIODIC_PATH=${CLEAN_BRANCH_NAME}-last-clean-periodic-test/last-ocir-pushed-verrazzano-bom.json 128 export SCAN_BOM_SNAPSHOT_PATH=${CLEAN_BRANCH_NAME}-last-snapshot/last-ocir-pushed-verrazzano-bom.json 129 export SCAN_BOM_FEATURE_PATH=${CLEAN_BRANCH_NAME}/last-ocir-pushed-verrazzano-bom.json 130 export SCAN_LAST_PERIODIC_BOM_FILE=${BOM_DIR}/${SCAN_BOM_PERIODIC_PATH} 131 export SCAN_LAST_SNAPSHOT_BOM_FILE=${BOM_DIR}/${SCAN_BOM_SNAPSHOT_PATH} 132 export SCAN_FEATURE_BOM_FILE=${BOM_DIR}/${SCAN_BOM_FEATURE_PATH} 133 export SCAN_COMMIT_PERIODIC_PATH=${CLEAN_BRANCH_NAME}-last-clean-periodic-test/verrazzano_periodic-commit.txt 134 export SCAN_LAST_PERIODIC_COMMIT_FILE=${BOM_DIR}/${SCAN_COMMIT_PERIODIC_PATH} 135 136 # If there is a periodic BOM file for this branch, get those results 137 GIT_COMMIT="TBD-Commit" 138 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${SCAN_BOM_PERIODIC_PATH} --file ${SCAN_LAST_PERIODIC_BOM_FILE} 2> /dev/null 139 if [ $? -eq 0 ]; then 140 echo "Fetching scan results for BOM: ${SCAN_LAST_PERIODIC_BOM_FILE}" 141 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${SCAN_COMMIT_PERIODIC_PATH} --file ${SCAN_LAST_PERIODIC_COMMIT_FILE} 2> /dev/null 142 if [ $? -eq 0 ]; then 143 GIT_COMMIT=$(cat ${SCAN_LAST_PERIODIC_COMMIT_FILE} | cut -d'=' -f2) 144 fi 145 export SCAN_RESULTS_DIR=${SCAN_RESULTS_BASE_DIR}/latest-periodic 146 mkdir -p ${SCAN_RESULTS_DIR} 147 ${RELEASE_SCRIPT_DIR}/scan_bom_images.sh -b ${SCAN_LAST_PERIODIC_BOM_FILE} -o ${SCAN_RESULTS_DIR} -r ${OCIR_SCAN_REGISTRY} -x ${OCIR_REPOSITORY_BASE} || exit 1 148 ${RELEASE_SCRIPT_DIR}/get_ocir_scan_results.sh ${SCAN_LAST_PERIODIC_BOM_FILE} || exit 1 149 ${RELEASE_SCRIPT_DIR}/generate_vulnerability_report.sh ${SCAN_RESULTS_DIR} ${GIT_COMMIT} ${CLEAN_BRANCH_NAME} "periodic" ${SCAN_DATETIME} ${BUILD_NUMBER} 150 ${RELEASE_SCRIPT_DIR}/generate_upload_file.sh ${SCAN_RESULTS_DIR}/consolidated.csv "periodic" > ${SCAN_RESULTS_DIR}/consolidated-upload.json 151 publish_results "last-clean-periodic-test" ${SCAN_LAST_PERIODIC_BOM_FILE} ${SCAN_RESULTS_DIR} 152 else 153 echo "INFO: Did not find a periodic BOM for ${CLEAN_BRANCH_NAME}" 154 rm ${SCAN_LAST_PERIODIC_BOM_FILE} || true 155 fi 156 157 # If there is a snapshot BOM file for this branch, get those results 158 GIT_COMMIT="TBD-Commit" 159 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${SCAN_BOM_SNAPSHOT_PATH} --file ${SCAN_LAST_SNAPSHOT_BOM_FILE} 2> /dev/null 160 if [ $? -eq 0 ]; then 161 echo "Fetching scan results for BOM: ${SCAN_LAST_SNAPSHOT_BOM_FILE}" 162 export SCAN_RESULTS_DIR=${SCAN_RESULTS_BASE_DIR}/last-snapshot-possibly-old 163 mkdir -p ${SCAN_RESULTS_DIR} 164 ${RELEASE_SCRIPT_DIR}/scan_bom_images.sh -b ${SCAN_LAST_SNAPSHOT_BOM_FILE} -o ${SCAN_RESULTS_DIR} -r ${OCIR_SCAN_REGISTRY} -x ${OCIR_REPOSITORY_BASE} || exit 1 165 ${RELEASE_SCRIPT_DIR}/get_ocir_scan_results.sh ${SCAN_LAST_SNAPSHOT_BOM_FILE} || exit 1 166 ${RELEASE_SCRIPT_DIR}/generate_vulnerability_report.sh ${SCAN_RESULTS_DIR} ${GIT_COMMIT} ${CLEAN_BRANCH_NAME} "snapshot" ${SCAN_DATETIME} ${BUILD_NUMBER} 167 ${RELEASE_SCRIPT_DIR}/generate_upload_file.sh ${SCAN_RESULTS_DIR}/consolidated.csv "snapshot" > ${SCAN_RESULTS_DIR}/consolidated-upload.json 168 publish_results "last-snapshot" ${SCAN_LAST_SNAPSHOT_BOM_FILE} ${SCAN_RESULTS_DIR} 169 else 170 echo "INFO: Did not find a snapshot BOM for ${CLEAN_BRANCH_NAME}" 171 rm ${SCAN_LAST_SNAPSHOT_BOM_FILE} || true 172 fi 173 174 # If this is a feature branch, get those results 175 GIT_COMMIT="TBD-Commit" 176 if [[ "${CLEAN_BRANCH_NAME}" != "master" ]] && [[ "${CLEAN_BRANCH_NAME}" != release-* ]]; then 177 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${SCAN_BOM_FEATURE_PATH} --file ${SCAN_FEATURE_BOM_FILE} 2> /dev/null 178 if [ $? -eq 0 ]; then 179 echo "Fetching scan results for BOM: ${SCAN_FEATURE_BOM_FILE}" 180 export SCAN_RESULTS_DIR=${SCAN_RESULTS_BASE_DIR}/feature-branch-latest 181 mkdir -p ${SCAN_RESULTS_DIR} 182 ${RELEASE_SCRIPT_DIR}/scan_bom_images.sh -b ${SCAN_FEATURE_BOM_FILE} -o ${SCAN_RESULTS_DIR} -r ${OCIR_SCAN_REGISTRY} -x ${OCIR_REPOSITORY_BASE} || exit 1 183 ${RELEASE_SCRIPT_DIR}/get_ocir_scan_results.sh ${SCAN_FEATURE_BOM_FILE} || exit 1 184 ${RELEASE_SCRIPT_DIR}/generate_vulnerability_report.sh ${SCAN_RESULTS_DIR} ${GIT_COMMIT} ${CLEAN_BRANCH_NAME} "feature" ${SCAN_DATETIME} ${BUILD_NUMBER} 185 ${RELEASE_SCRIPT_DIR}/generate_upload_file.sh ${SCAN_RESULTS_DIR}/consolidated.csv "feature" > ${SCAN_RESULTS_DIR}/consolidated-upload.json 186 publish_results "feature" ${SCAN_FEATURE_BOM_FILE} ${SCAN_RESULTS_DIR} 187 else 188 echo "INFO: Did not find a feature BOM for ${CLEAN_BRANCH_NAME}" 189 rm ${SCAN_FEATURE_BOM_FILE} || true 190 fi 191 fi 192 193 if [[ "${CLEAN_BRANCH_NAME}" == release-* ]]; then 194 # Get the list of matching releases, for example, on branch "release-1.0" the matching releases are "v1.0.0", "v1.0.1", ... 195 echo "Attempting to fetch BOMs for released versions on branch: ${CLEAN_BRANCH_NAME}" 196 197 MAJOR_MINOR_VERSION=${CLEAN_BRANCH_NAME:8} 198 VERSIONS=$(gh release list | cut -f 3 | grep v${MAJOR_MINOR_VERSION}) 199 200 # We only are using the latest version. The "gh release list" returns them in order from most recent first, so the first 201 # version returned in the list is the most recent release version for the release-* branch. 202 # If this is a new release, there will not be any released version found yet 203 echo "All versions found: ${VERSIONS}" 204 205 if [ ! -z ${VERSIONS} ]; then 206 VERSION=$(echo $VERSIONS | cut -d ' ' -f 1) 207 208 GIT_COMMIT=$(git rev-list -n 1 ${VERSION}) 209 echo "Fetching BOM for ${VERSION}" 210 export SCAN_BOM_FILE=${BOM_DIR}/${VERSION}-bom.json 211 get_bom_from_release ${VERSION} ${SCAN_BOM_FILE} 212 export SCAN_RESULTS_DIR=${SCAN_RESULTS_BASE_DIR}/${VERSION} 213 mkdir -p ${SCAN_RESULTS_DIR} 214 215 echo "Fetching scan results for BOM: ${SCAN_BOM_FILE}" 216 ${RELEASE_SCRIPT_DIR}/scan_bom_images.sh -b ${SCAN_BOM_FILE} -o ${SCAN_RESULTS_DIR} -r ${OCIR_SCAN_REGISTRY} -x ${OCIR_REPOSITORY_BASE} 217 ${RELEASE_SCRIPT_DIR}/get_ocir_scan_results.sh ${SCAN_BOM_FILE} 218 ${RELEASE_SCRIPT_DIR}/generate_vulnerability_report.sh ${SCAN_RESULTS_DIR} ${GIT_COMMIT} ${CLEAN_BRANCH_NAME} ${VERSION} ${SCAN_DATETIME} ${BUILD_NUMBER} 219 ${RELEASE_SCRIPT_DIR}/generate_upload_file.sh ${SCAN_RESULTS_DIR}/consolidated.csv "${VERSION}" > ${SCAN_RESULTS_DIR}/consolidated-upload.json 220 publish_results ${VERSION} ${SCAN_BOM_FILE} ${SCAN_RESULTS_DIR} 221 else 222 echo "There are no released versions for v${MAJOR_MINOR_VERSION} yet" 223 fi 224 fi