github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/config/scripts/create_ocir_repositories.sh (about)

     1  #!/bin/bash
     2  # Copyright (c) 2021, 2023, Oracle and/or its affiliates.
     3  # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     4  #
     5  # Create a Docker repository in a specific compartment using an exploded tarball of Verrazzano images; useful for
     6  # scoping a repo someplace other than the root compartment.
     7  #
     8  set -u
     9  
    10  usage() {
    11    echo """
    12  Create OCIR repos for a set of exported Docker images tarballs located in a local directory in a specific
    13  tenancy compartment.  This script reads the image repo information out of each tar file and uses that to create
    14  a corresponding Docker repo in the target tenancy compartment, under that tenancy's namespace.  You can provide either
    15  the full region name (e.g., "us-phoenix-1", or the region short name (e.g., "phx").
    16  
    17  See https://docs.oracle.com/en-us/iaas/Content/Registry/Tasks/registrycreatingarepository.htm for details on
    18  repository creation in an OCI tenancy.
    19  
    20  Usage:
    21  
    22  $0  -p <parent-repo> -c <compartment-id> [ -r <region> -d <path> ]
    23  
    24  -r Region name (e.g., "us-phoenix-1")
    25  -s Region short name (e.g., "phx", "lhr")
    26  -p Parent repo, without the tenancy namespace
    27  -c Compartment ID
    28  -d Images directory
    29  -t Target ID. This is an existing Target-Id used for OCIR scanning. Repositories which are not already targeted there
    30     will be created as PRIVATE-ONLY and will also added to the target list for this existing Target Id. If not found
    31     this will fail. Note that repositories which are already targeted will be skipped for creation (they already exist).
    32     This is used by the CI for OCIR scanning setup.
    33  
    34  Example, to create a repo in compartment ocid.compartment.oc1..blah, where the desired docker path with tenancy namespace
    35  to the image is "myreporoot/testuser/myrepo/v8o", and the extracted tarball location is /tmp/exploded:
    36  
    37  $0 -p myreporoot/testuser/myrepo/v8o -r uk-london-1 -c ocid.compartment.oc1..blah -d /tmp/exploded
    38    """
    39  }
    40  
    41  # Function to check the env for OCIR scan related functions
    42  function checkEnvForScan() {
    43    if [ -z $OCIR_SCAN_TARGET_ID ]; then
    44      echo "OCIR_SCAN_TARGET_ID is required to be defined"
    45      return 1
    46    fi
    47  
    48    if [ -z $REGION ]; then
    49      echo "REGION is required to be defined"
    50      return 1
    51    fi
    52    return 0
    53  }
    54  
    55  # getTargetJson
    56  function getTargetJson() {
    57    checkEnvForScan
    58    if [ $? -ne 0 ]; then
    59      return 1
    60    fi
    61  
    62    if [ -z $1 ]; then
    63      echo "Please specify the name of the environment variable to return the target json filename into"
    64      return 1
    65    fi
    66    local  __resultvar=$1
    67  
    68    local targetfile=$(mktemp temp-target-XXXXXX.json)
    69    oci vulnerability-scanning container scan target get --container-scan-target-id $OCIR_SCAN_TARGET_ID --region ${REGION} > $targetfile
    70    if [ $? -ne 0 ]; then
    71      echo "Failed to get target $OCIR_SCAN_TARGET_ID in $REGION"
    72      return 1
    73    fi
    74  
    75    # Set the target output filename into the supplied environment variable
    76    eval $__resultvar="'$targetfile'"
    77    return 0
    78  }
    79  
    80  # addNewRepositoriesToTarget:  This will update the existing target by adding new repositories into it
    81  #   $1 array of repositories to add
    82  #   $2 target.json
    83  function addNewRepositoriesToTarget() {
    84    checkEnvForScan
    85    local retval=0
    86    if [ $? -ne 0 ]; then
    87      return 1
    88    fi
    89  
    90    if [ -z $1 ] || [ -z $2 ] || [ ! -f $2 ]; then
    91      echo "Invalid arguments supplied to addNewRepositoriesToTarget"
    92      return "error"
    93    fi
    94  
    95    # REVIEW: If we can give the array to jq all at once that will be nice, but for now
    96    # just adding one element at a time here
    97    local newtargetfile=$(mktemp temp-new-target-XXXXXX.json)
    98    cp $2 $newtargetfile
    99    read -ra repo_array <<< $1
   100    for repo_name in "${repo_array[@]}"
   101    do
   102      echo $(jq --arg repo "$repo_name" '.data."target-registry".repositories += [$repo]' $newtargetfile | jq '.') > $newtargetfile
   103      if [ $? -ne 0 ]; then
   104        echo "Problem adding new repositories into the existing Target"
   105        retval=1
   106      fi
   107    done
   108  
   109    if [ $retval -eq 0 ]; then
   110      cat $newtargetfile
   111      # Update the target using the new target file
   112      oci vulnerability-scanning container scan target update --from-json file://$newtargetfile --container-scan-target-id $OCIR_SCAN_TARGET_ID --region ${REGION}
   113      if [ $? -ne 0 ]; then
   114        echo "Problem updating the Target"
   115        retval=1
   116      fi
   117    fi
   118    rm $newtargetfile
   119    return $retval
   120  }
   121  
   122  # isRepositoryTargeted. returns true if found in the list, false if not found, and error if arguments are invalid
   123  #  $1 name of repository
   124  #  $2 target.json
   125  function isRepositoryTargeted() {
   126    if [ -z $1 ] || [ -z $2 ] || [ ! -f $2 ]; then
   127      echo "Invalid arguments supplied to isRepositoryTargeted"
   128      return 2
   129    fi
   130    grep $1 $2 > /dev/null
   131    return $?
   132  }
   133  
   134  # Main driver for processing images from a locally downloaded set of tarballs
   135  function create_image_repos_from_archives() {
   136    declare -a added_repositories=()
   137    local target_file=""
   138  
   139    # If we have a scan target, and if it is usable.
   140    # NOTE: We are having issues with the scan target getting into a bad state, the lookup can start failing
   141    # with a 404. If that happens we proceed without it and do NOT fail.
   142    local target_accessed="false"
   143    if [ ! -z $OCIR_SCAN_TARGET_ID ]; then
   144      getTargetJson "target_file"
   145      if [ $? -eq 0 ]; then
   146        echo "Target JSON was retrieved"
   147        target_accessed="true"
   148      else
   149        echo "No target JSON was retrieved, target related operations will be skipped but other processing will proceed based on target id being specified"
   150      fi
   151    fi
   152  
   153    local repositories_listed="false"
   154    local reposfile=$(mktemp temp-repositories-XXXXXX.json)
   155    oci --region ${REGION} artifacts container repository list --compartment-id ${COMPARTMENT_ID} --all > $reposfile
   156    if [ $? -eq 0 ]; then
   157      echo "Repositories listed"
   158      repositories_listed="true"
   159    else
   160      echo "Unable to list the existing repositories to check for existence"
   161    fi
   162  
   163    # Loop through tar files
   164    local cmdoutfile=$(mktemp temp-oci-cli-XXXXXX.out)
   165    echo "Using local image downloads"
   166    for file in ${IMAGES_DIR}/*.tar; do
   167      echo "Processing file ${file}"
   168      if [ ! -e ${file} ]; then
   169        echo "Image tar file ${file} does not exist!"
   170        exit 1
   171      fi
   172  
   173      # Build up the image name and target image names, and create the repo
   174      local from_image=$(tar xOvf $file manifest.json | jq -r '.[0].RepoTags[0]')
   175      local from_image_name=$(basename $from_image | cut -d \: -f 1)
   176      local from_repository=$(dirname $from_image | cut -d \/ -f 2-)
   177  
   178      # When OCIR_SCAN_TARGET_ID is set, all of the repositories used for scanning are created as private (we only use them for scanning)
   179      local is_public="false"
   180      if [[ "$from_image_name" =~ rancher.* ]] || [[ "$from_repository" == "verrazzano/rancher" ]] \
   181        || [[ "$from_image_name" == "verrazzano-platform-operator" ]] || [[ "$from_image_name" =~ cluster-api.* ]] \
   182        && [[ -z $OCIR_SCAN_TARGET_ID ]]; then
   183        # Rancher repos must be public
   184        is_public="true"
   185      fi
   186  
   187      local repo_path=${from_repository}/${from_image_name}
   188      if [ -n "${PARENT_REPO}" ]; then
   189        repo_path=${PARENT_REPO}/${repo_path}
   190      fi
   191  
   192      # If we have a scan target, we can see if it is targeted already (if it is we can skip creating it)
   193      # if not we track the ones we go ahead and create so we can target them afterwards
   194      if [ ! -z $OCIR_SCAN_TARGET_ID ]; then
   195        # Only try target related operations if we were able to access the target, skip otherwise
   196        if [ "$target_accessed" == "true" ]; then
   197          isRepositoryTargeted $repo_path $target_file
   198          local repository_targeted=$?
   199          if [ $repository_targeted -eq 2 ]; then
   200            echo "Error checking if repository was targeted ${repo_path}"
   201            rm $target_file
   202            exit 1
   203          fi
   204          # If it is targeted already, then it exists and we skip creating a new repository
   205          if [ $repository_targeted -eq 0 ]; then
   206            echo "$repo_path is already targeted"
   207            continue
   208          fi
   209          # If we got here, we will add it to the list to try to target it
   210          added_repositories+=("$repo_path")
   211          echo "$repo_path needs to be targeted"
   212        else
   213          echo "skipping target checking for $repo_path (target not accessible)"
   214        fi
   215  
   216        # Check if it exists already first
   217        if [ "$repositories_listed" == "false" ]; then
   218          echo "skipping existing repository check for $repo_path (repositories unable to be listed)"
   219          continue
   220        fi
   221  
   222        grep "${repo_path}" $reposfile > /dev/null
   223        if [ $? -eq 0 ]; then
   224          echo "$repo_path already exists and doesn't need to be created"
   225          continue
   226        fi
   227        echo "$repo_path needs to be created"
   228      fi
   229  
   230      echo "Creating repository ${repo_path} in ${REGION}, public: ${is_public}"
   231      oci --region ${REGION} artifacts container repository create --display-name ${repo_path} \
   232        --is-public ${is_public} --compartment-id ${COMPARTMENT_ID} &> $cmdoutfile
   233      if [ $? -ne 0 ]; then
   234        grep 'Repository already exists' $cmdoutfile
   235        if [ $? -ne 0 ]; then
   236          echo "Repository creation failed"
   237          exit 1
   238        else
   239          echo "Repository already existed"
   240        fi
   241      fi
   242    done
   243  
   244    # If we added new repositories, we need to get them added to the target so they will get scanned
   245    if [ ! -z $OCIR_SCAN_TARGET_ID ] && [ "$target_accessed" == "true" ]; then
   246      # FIXME: Do not enable until we are sure the VSS lifecycle state issues with update are understood and handled
   247      #  addNewRepositoriesToTarget "${added_repositories}" $target_file
   248      rm $target_file || true
   249    fi
   250  
   251    if [ "$repositories_listed" == "true" ]; then
   252      rm $reposfile || true
   253    fi
   254  }
   255  
   256  IMAGES_DIR=.
   257  REGION=""
   258  REGION_SHORT_NAME=""
   259  OCIR_SCAN_TARGET_ID=""
   260  COMPARTMENT_ID=""
   261  PARENT_REPO=""
   262  
   263  while getopts ":s:c:r:p:d:t:" opt; do
   264    case ${opt} in
   265    c) # compartment ID
   266      COMPARTMENT_ID=${OPTARG}
   267      ;;
   268    r) # region
   269      REGION=${OPTARG}
   270      ;;
   271    p) # parent repo
   272      PARENT_REPO=${OPTARG}
   273      ;;
   274    d) # images dir
   275      IMAGES_DIR="${OPTARG}"
   276      ;;
   277    s )
   278      REGION_SHORT_NAME="${OPTARG}"
   279      ;;
   280    t )
   281      OCIR_SCAN_TARGET_ID="${OPTARG}"
   282      ;;
   283    \?)
   284      usage
   285      ;;
   286    esac
   287  done
   288  shift $((OPTIND - 1))
   289  
   290  if [ -z "${REGION}" ]; then
   291    if [ -z "${REGION_SHORT_NAME}" ]; then
   292      echo "Must provide either the full or the short region name"
   293      usage
   294      exit 1
   295    fi
   296    echo "REGION_SHORT_NAME=$REGION_SHORT_NAME"
   297    REGION=$(oci --region us-phoenix-1 iam region list | jq -r  --arg regionAbbr ${REGION_SHORT_NAME} '.data[] | select(.key|test($regionAbbr;"i")) | .name')
   298    if [ -z "${REGION}" ] || [ "null" == "${REGION}" ]; then
   299      echo "Invalid short region name ${REGION_SHORT_NAME}"
   300      usage
   301      exit 1
   302    fi
   303  fi
   304  
   305  if [ -z "${PARENT_REPO}" ]; then
   306    echo "Repository pattern not provided"
   307    usage
   308    exit 1
   309  fi
   310  if [ -z "${COMPARTMENT_ID}" ]; then
   311    echo "Compartment ID not provided"
   312    usage
   313    exit 1
   314  fi
   315  
   316  create_image_repos_from_archives