github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/config/scripts/create_ocir_repositories.sh (about) 1 #!/bin/bash 2 # Copyright (c) 2021, 2023, Oracle and/or its affiliates. 3 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 4 # 5 # Create a Docker repository in a specific compartment using an exploded tarball of Verrazzano images; useful for 6 # scoping a repo someplace other than the root compartment. 7 # 8 set -u 9 10 usage() { 11 echo """ 12 Create OCIR repos for a set of exported Docker images tarballs located in a local directory in a specific 13 tenancy compartment. This script reads the image repo information out of each tar file and uses that to create 14 a corresponding Docker repo in the target tenancy compartment, under that tenancy's namespace. You can provide either 15 the full region name (e.g., "us-phoenix-1", or the region short name (e.g., "phx"). 16 17 See https://docs.oracle.com/en-us/iaas/Content/Registry/Tasks/registrycreatingarepository.htm for details on 18 repository creation in an OCI tenancy. 19 20 Usage: 21 22 $0 -p <parent-repo> -c <compartment-id> [ -r <region> -d <path> ] 23 24 -r Region name (e.g., "us-phoenix-1") 25 -s Region short name (e.g., "phx", "lhr") 26 -p Parent repo, without the tenancy namespace 27 -c Compartment ID 28 -d Images directory 29 -t Target ID. This is an existing Target-Id used for OCIR scanning. Repositories which are not already targeted there 30 will be created as PRIVATE-ONLY and will also added to the target list for this existing Target Id. If not found 31 this will fail. Note that repositories which are already targeted will be skipped for creation (they already exist). 32 This is used by the CI for OCIR scanning setup. 33 34 Example, to create a repo in compartment ocid.compartment.oc1..blah, where the desired docker path with tenancy namespace 35 to the image is "myreporoot/testuser/myrepo/v8o", and the extracted tarball location is /tmp/exploded: 36 37 $0 -p myreporoot/testuser/myrepo/v8o -r uk-london-1 -c ocid.compartment.oc1..blah -d /tmp/exploded 38 """ 39 } 40 41 # Function to check the env for OCIR scan related functions 42 function checkEnvForScan() { 43 if [ -z $OCIR_SCAN_TARGET_ID ]; then 44 echo "OCIR_SCAN_TARGET_ID is required to be defined" 45 return 1 46 fi 47 48 if [ -z $REGION ]; then 49 echo "REGION is required to be defined" 50 return 1 51 fi 52 return 0 53 } 54 55 # getTargetJson 56 function getTargetJson() { 57 checkEnvForScan 58 if [ $? -ne 0 ]; then 59 return 1 60 fi 61 62 if [ -z $1 ]; then 63 echo "Please specify the name of the environment variable to return the target json filename into" 64 return 1 65 fi 66 local __resultvar=$1 67 68 local targetfile=$(mktemp temp-target-XXXXXX.json) 69 oci vulnerability-scanning container scan target get --container-scan-target-id $OCIR_SCAN_TARGET_ID --region ${REGION} > $targetfile 70 if [ $? -ne 0 ]; then 71 echo "Failed to get target $OCIR_SCAN_TARGET_ID in $REGION" 72 return 1 73 fi 74 75 # Set the target output filename into the supplied environment variable 76 eval $__resultvar="'$targetfile'" 77 return 0 78 } 79 80 # addNewRepositoriesToTarget: This will update the existing target by adding new repositories into it 81 # $1 array of repositories to add 82 # $2 target.json 83 function addNewRepositoriesToTarget() { 84 checkEnvForScan 85 local retval=0 86 if [ $? -ne 0 ]; then 87 return 1 88 fi 89 90 if [ -z $1 ] || [ -z $2 ] || [ ! -f $2 ]; then 91 echo "Invalid arguments supplied to addNewRepositoriesToTarget" 92 return "error" 93 fi 94 95 # REVIEW: If we can give the array to jq all at once that will be nice, but for now 96 # just adding one element at a time here 97 local newtargetfile=$(mktemp temp-new-target-XXXXXX.json) 98 cp $2 $newtargetfile 99 read -ra repo_array <<< $1 100 for repo_name in "${repo_array[@]}" 101 do 102 echo $(jq --arg repo "$repo_name" '.data."target-registry".repositories += [$repo]' $newtargetfile | jq '.') > $newtargetfile 103 if [ $? -ne 0 ]; then 104 echo "Problem adding new repositories into the existing Target" 105 retval=1 106 fi 107 done 108 109 if [ $retval -eq 0 ]; then 110 cat $newtargetfile 111 # Update the target using the new target file 112 oci vulnerability-scanning container scan target update --from-json file://$newtargetfile --container-scan-target-id $OCIR_SCAN_TARGET_ID --region ${REGION} 113 if [ $? -ne 0 ]; then 114 echo "Problem updating the Target" 115 retval=1 116 fi 117 fi 118 rm $newtargetfile 119 return $retval 120 } 121 122 # isRepositoryTargeted. returns true if found in the list, false if not found, and error if arguments are invalid 123 # $1 name of repository 124 # $2 target.json 125 function isRepositoryTargeted() { 126 if [ -z $1 ] || [ -z $2 ] || [ ! -f $2 ]; then 127 echo "Invalid arguments supplied to isRepositoryTargeted" 128 return 2 129 fi 130 grep $1 $2 > /dev/null 131 return $? 132 } 133 134 # Main driver for processing images from a locally downloaded set of tarballs 135 function create_image_repos_from_archives() { 136 declare -a added_repositories=() 137 local target_file="" 138 139 # If we have a scan target, and if it is usable. 140 # NOTE: We are having issues with the scan target getting into a bad state, the lookup can start failing 141 # with a 404. If that happens we proceed without it and do NOT fail. 142 local target_accessed="false" 143 if [ ! -z $OCIR_SCAN_TARGET_ID ]; then 144 getTargetJson "target_file" 145 if [ $? -eq 0 ]; then 146 echo "Target JSON was retrieved" 147 target_accessed="true" 148 else 149 echo "No target JSON was retrieved, target related operations will be skipped but other processing will proceed based on target id being specified" 150 fi 151 fi 152 153 local repositories_listed="false" 154 local reposfile=$(mktemp temp-repositories-XXXXXX.json) 155 oci --region ${REGION} artifacts container repository list --compartment-id ${COMPARTMENT_ID} --all > $reposfile 156 if [ $? -eq 0 ]; then 157 echo "Repositories listed" 158 repositories_listed="true" 159 else 160 echo "Unable to list the existing repositories to check for existence" 161 fi 162 163 # Loop through tar files 164 local cmdoutfile=$(mktemp temp-oci-cli-XXXXXX.out) 165 echo "Using local image downloads" 166 for file in ${IMAGES_DIR}/*.tar; do 167 echo "Processing file ${file}" 168 if [ ! -e ${file} ]; then 169 echo "Image tar file ${file} does not exist!" 170 exit 1 171 fi 172 173 # Build up the image name and target image names, and create the repo 174 local from_image=$(tar xOvf $file manifest.json | jq -r '.[0].RepoTags[0]') 175 local from_image_name=$(basename $from_image | cut -d \: -f 1) 176 local from_repository=$(dirname $from_image | cut -d \/ -f 2-) 177 178 # When OCIR_SCAN_TARGET_ID is set, all of the repositories used for scanning are created as private (we only use them for scanning) 179 local is_public="false" 180 if [[ "$from_image_name" =~ rancher.* ]] || [[ "$from_repository" == "verrazzano/rancher" ]] \ 181 || [[ "$from_image_name" == "verrazzano-platform-operator" ]] || [[ "$from_image_name" =~ cluster-api.* ]] \ 182 && [[ -z $OCIR_SCAN_TARGET_ID ]]; then 183 # Rancher repos must be public 184 is_public="true" 185 fi 186 187 local repo_path=${from_repository}/${from_image_name} 188 if [ -n "${PARENT_REPO}" ]; then 189 repo_path=${PARENT_REPO}/${repo_path} 190 fi 191 192 # If we have a scan target, we can see if it is targeted already (if it is we can skip creating it) 193 # if not we track the ones we go ahead and create so we can target them afterwards 194 if [ ! -z $OCIR_SCAN_TARGET_ID ]; then 195 # Only try target related operations if we were able to access the target, skip otherwise 196 if [ "$target_accessed" == "true" ]; then 197 isRepositoryTargeted $repo_path $target_file 198 local repository_targeted=$? 199 if [ $repository_targeted -eq 2 ]; then 200 echo "Error checking if repository was targeted ${repo_path}" 201 rm $target_file 202 exit 1 203 fi 204 # If it is targeted already, then it exists and we skip creating a new repository 205 if [ $repository_targeted -eq 0 ]; then 206 echo "$repo_path is already targeted" 207 continue 208 fi 209 # If we got here, we will add it to the list to try to target it 210 added_repositories+=("$repo_path") 211 echo "$repo_path needs to be targeted" 212 else 213 echo "skipping target checking for $repo_path (target not accessible)" 214 fi 215 216 # Check if it exists already first 217 if [ "$repositories_listed" == "false" ]; then 218 echo "skipping existing repository check for $repo_path (repositories unable to be listed)" 219 continue 220 fi 221 222 grep "${repo_path}" $reposfile > /dev/null 223 if [ $? -eq 0 ]; then 224 echo "$repo_path already exists and doesn't need to be created" 225 continue 226 fi 227 echo "$repo_path needs to be created" 228 fi 229 230 echo "Creating repository ${repo_path} in ${REGION}, public: ${is_public}" 231 oci --region ${REGION} artifacts container repository create --display-name ${repo_path} \ 232 --is-public ${is_public} --compartment-id ${COMPARTMENT_ID} &> $cmdoutfile 233 if [ $? -ne 0 ]; then 234 grep 'Repository already exists' $cmdoutfile 235 if [ $? -ne 0 ]; then 236 echo "Repository creation failed" 237 exit 1 238 else 239 echo "Repository already existed" 240 fi 241 fi 242 done 243 244 # If we added new repositories, we need to get them added to the target so they will get scanned 245 if [ ! -z $OCIR_SCAN_TARGET_ID ] && [ "$target_accessed" == "true" ]; then 246 # FIXME: Do not enable until we are sure the VSS lifecycle state issues with update are understood and handled 247 # addNewRepositoriesToTarget "${added_repositories}" $target_file 248 rm $target_file || true 249 fi 250 251 if [ "$repositories_listed" == "true" ]; then 252 rm $reposfile || true 253 fi 254 } 255 256 IMAGES_DIR=. 257 REGION="" 258 REGION_SHORT_NAME="" 259 OCIR_SCAN_TARGET_ID="" 260 COMPARTMENT_ID="" 261 PARENT_REPO="" 262 263 while getopts ":s:c:r:p:d:t:" opt; do 264 case ${opt} in 265 c) # compartment ID 266 COMPARTMENT_ID=${OPTARG} 267 ;; 268 r) # region 269 REGION=${OPTARG} 270 ;; 271 p) # parent repo 272 PARENT_REPO=${OPTARG} 273 ;; 274 d) # images dir 275 IMAGES_DIR="${OPTARG}" 276 ;; 277 s ) 278 REGION_SHORT_NAME="${OPTARG}" 279 ;; 280 t ) 281 OCIR_SCAN_TARGET_ID="${OPTARG}" 282 ;; 283 \?) 284 usage 285 ;; 286 esac 287 done 288 shift $((OPTIND - 1)) 289 290 if [ -z "${REGION}" ]; then 291 if [ -z "${REGION_SHORT_NAME}" ]; then 292 echo "Must provide either the full or the short region name" 293 usage 294 exit 1 295 fi 296 echo "REGION_SHORT_NAME=$REGION_SHORT_NAME" 297 REGION=$(oci --region us-phoenix-1 iam region list | jq -r --arg regionAbbr ${REGION_SHORT_NAME} '.data[] | select(.key|test($regionAbbr;"i")) | .name') 298 if [ -z "${REGION}" ] || [ "null" == "${REGION}" ]; then 299 echo "Invalid short region name ${REGION_SHORT_NAME}" 300 usage 301 exit 1 302 fi 303 fi 304 305 if [ -z "${PARENT_REPO}" ]; then 306 echo "Repository pattern not provided" 307 usage 308 exit 1 309 fi 310 if [ -z "${COMPARTMENT_ID}" ]; then 311 echo "Compartment ID not provided" 312 usage 313 exit 1 314 fi 315 316 create_image_repos_from_archives