github.com/rawahars/moby@v24.0.4+incompatible/contrib/download-frozen-image-v2.sh (about) 1 #!/usr/bin/env bash 2 set -eo pipefail 3 4 # hello-world latest ef872312fe1b 3 months ago 910 B 5 # hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B 6 7 # debian latest f6fab3b798be 10 weeks ago 85.1 MB 8 # debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB 9 10 # check if essential commands are in our PATH 11 for cmd in curl jq; do 12 if ! command -v $cmd &> /dev/null; then 13 echo >&2 "error: \"$cmd\" not found!" 14 exit 1 15 fi 16 done 17 18 usage() { 19 echo "usage: $0 dir image[:tag][@digest] ..." 20 echo " $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7" 21 [ -z "$1" ] || exit "$1" 22 } 23 24 dir="$1" # dir for building tar in 25 shift || usage 1 >&2 26 27 if ! [ $# -gt 0 ] && [ "$dir" ]; then 28 usage 2 >&2 29 fi 30 mkdir -p "$dir" 31 32 # hacky workarounds for Bash 3 support (no associative arrays) 33 images=() 34 rm -f "$dir"/tags-*.tmp 35 manifestJsonEntries=() 36 doNotGenerateManifestJson= 37 # repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."' 38 39 # bash v4 on Windows CI requires CRLF separator... and linux doesn't seem to care either way 40 newlineIFS=$'\n' 41 major=$(echo "${BASH_VERSION%%[^0.9]}" | cut -d. -f1) 42 if [ "$major" -ge 4 ]; then 43 newlineIFS=$'\r\n' 44 fi 45 46 registryBase='https://registry-1.docker.io' 47 authBase='https://auth.docker.io' 48 authService='registry.docker.io' 49 50 # https://github.com/moby/moby/issues/33700 51 fetch_blob() { 52 local token="$1" 53 shift 54 local image="$1" 55 shift 56 local digest="$1" 57 shift 58 local targetFile="$1" 59 shift 60 local curlArgs=("$@") 61 62 local curlHeaders 63 curlHeaders="$( 64 curl -S "${curlArgs[@]}" \ 65 -H "Authorization: Bearer $token" \ 66 "$registryBase/v2/$image/blobs/$digest" \ 67 -o "$targetFile" \ 68 -D- 69 )" 70 curlHeaders="$(echo "$curlHeaders" | tr -d '\r')" 71 if grep -qE "^HTTP/[0-9].[0-9] 3" <<< "$curlHeaders"; then 72 rm -f "$targetFile" 73 74 local blobRedirect 75 blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')" 76 if [ -z "$blobRedirect" ]; then 77 echo >&2 "error: failed fetching '$image' blob '$digest'" 78 echo "$curlHeaders" | head -1 >&2 79 return 1 80 fi 81 82 curl -fSL "${curlArgs[@]}" \ 83 "$blobRedirect" \ 84 -o "$targetFile" 85 fi 86 } 87 88 # handle 'application/vnd.docker.distribution.manifest.v2+json' manifest 89 handle_single_manifest_v2() { 90 local manifestJson="$1" 91 shift 92 93 local configDigest 94 configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')" 95 local imageId="${configDigest#*:}" # strip off "sha256:" 96 97 local configFile="$imageId.json" 98 fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s 99 100 local layersFs 101 layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" 102 local IFS="$newlineIFS" 103 local layers 104 mapfile -t layers <<< "$layersFs" 105 unset IFS 106 107 echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." 108 local layerId= 109 local layerFiles=() 110 for i in "${!layers[@]}"; do 111 local layerMeta="${layers[$i]}" 112 113 local layerMediaType 114 layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')" 115 local layerDigest 116 layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')" 117 118 # save the previous layer's ID 119 local parentId="$layerId" 120 # create a new fake layer ID based on this layer's digest and the previous layer's fake ID 121 layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)" 122 # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value) 123 124 mkdir -p "$dir/$layerId" 125 echo '1.0' > "$dir/$layerId/VERSION" 126 127 if [ ! -s "$dir/$layerId/json" ]; then 128 local parentJson 129 parentJson="$(printf ', parent: "%s"' "$parentId")" 130 local addJson 131 addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")" 132 # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers 133 jq "$addJson + ." > "$dir/$layerId/json" <<- 'EOJSON' 134 { 135 "created": "0001-01-01T00:00:00Z", 136 "container_config": { 137 "Hostname": "", 138 "Domainname": "", 139 "User": "", 140 "AttachStdin": false, 141 "AttachStdout": false, 142 "AttachStderr": false, 143 "Tty": false, 144 "OpenStdin": false, 145 "StdinOnce": false, 146 "Env": null, 147 "Cmd": null, 148 "Image": "", 149 "Volumes": null, 150 "WorkingDir": "", 151 "Entrypoint": null, 152 "OnBuild": null, 153 "Labels": null 154 } 155 } 156 EOJSON 157 fi 158 159 case "$layerMediaType" in 160 application/vnd.docker.image.rootfs.diff.tar.gzip) 161 local layerTar="$layerId/layer.tar" 162 layerFiles=("${layerFiles[@]}" "$layerTar") 163 # TODO figure out why "-C -" doesn't work here 164 # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." 165 # "HTTP/1.1 416 Requested Range Not Satisfiable" 166 if [ -f "$dir/$layerTar" ]; then 167 # TODO hackpatch for no -C support :'( 168 echo "skipping existing ${layerId:0:12}" 169 continue 170 fi 171 local token 172 token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" 173 fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress-bar 174 ;; 175 176 *) 177 echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'" 178 exit 1 179 ;; 180 esac 181 done 182 183 # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons) 184 imageId="$layerId" 185 186 # munge the top layer image manifest to have the appropriate image configuration for older daemons 187 local imageOldConfig 188 imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")" 189 jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json" 190 191 local manifestJsonEntry 192 manifestJsonEntry="$( 193 echo '{}' | jq --raw-output '. + { 194 Config: "'"$configFile"'", 195 RepoTags: ["'"${image#library\/}:$tag"'"], 196 Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"' 197 }' 198 )" 199 manifestJsonEntries=("${manifestJsonEntries[@]}" "$manifestJsonEntry") 200 } 201 202 get_target_arch() { 203 if [ -n "${TARGETARCH:-}" ]; then 204 echo "${TARGETARCH}" 205 return 0 206 fi 207 208 if type go > /dev/null; then 209 go env GOARCH 210 return 0 211 fi 212 213 if type dpkg > /dev/null; then 214 debArch="$(dpkg --print-architecture)" 215 case "${debArch}" in 216 armel | armhf) 217 echo "arm" 218 return 0 219 ;; 220 *64el) 221 echo "${debArch%el}le" 222 return 0 223 ;; 224 *) 225 echo "${debArch}" 226 return 0 227 ;; 228 esac 229 fi 230 231 if type uname > /dev/null; then 232 uArch="$(uname -m)" 233 case "${uArch}" in 234 x86_64) 235 echo amd64 236 return 0 237 ;; 238 arm | armv[0-9]*) 239 echo arm 240 return 0 241 ;; 242 aarch64) 243 echo arm64 244 return 0 245 ;; 246 mips*) 247 echo >&2 "I see you are running on mips but I don't know how to determine endianness yet, so I cannot select a correct arch to fetch." 248 echo >&2 "Consider installing \"go\" on the system which I can use to determine the correct arch or specify it explicitly by setting TARGETARCH" 249 exit 1 250 ;; 251 *) 252 echo "${uArch}" 253 return 0 254 ;; 255 esac 256 257 fi 258 259 # default value 260 echo >&2 "Unable to determine CPU arch, falling back to amd64. You can specify a target arch by setting TARGETARCH" 261 echo amd64 262 } 263 264 get_target_variant() { 265 echo "${TARGETVARIANT:-}" 266 } 267 268 while [ $# -gt 0 ]; do 269 imageTag="$1" 270 shift 271 image="${imageTag%%[:@]*}" 272 imageTag="${imageTag#*:}" 273 digest="${imageTag##*@}" 274 tag="${imageTag%%@*}" 275 276 # add prefix library if passed official image 277 if [[ "$image" != *"/"* ]]; then 278 image="library/$image" 279 fi 280 281 imageFile="${image//\//_}" # "/" can't be in filenames :) 282 283 token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" 284 285 manifestJson="$( 286 curl -fsSL \ 287 -H "Authorization: Bearer $token" \ 288 -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ 289 -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ 290 -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ 291 "$registryBase/v2/$image/manifests/$digest" 292 )" 293 if [ "${manifestJson:0:1}" != '{' ]; then 294 echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" 295 echo >&2 " $manifestJson" 296 exit 1 297 fi 298 299 imageIdentifier="$image:$tag@$digest" 300 301 schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')" 302 case "$schemaVersion" in 303 2) 304 mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')" 305 306 case "$mediaType" in 307 application/vnd.docker.distribution.manifest.v2+json) 308 handle_single_manifest_v2 "$manifestJson" 309 ;; 310 application/vnd.docker.distribution.manifest.list.v2+json) 311 layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')" 312 IFS="$newlineIFS" 313 mapfile -t layers <<< "$layersFs" 314 unset IFS 315 316 found="" 317 targetArch="$(get_target_arch)" 318 targetVariant="$(get_target_variant)" 319 # parse first level multi-arch manifest 320 for i in "${!layers[@]}"; do 321 layerMeta="${layers[$i]}" 322 maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')" 323 maniVariant="$(echo "$layerMeta" | jq --raw-output '.platform.variant')" 324 if [[ "$maniArch" = "${targetArch}" ]] && [[ -z "${targetVariant}" || "$maniVariant" = "${targetVariant}" ]]; then 325 digest="$(echo "$layerMeta" | jq --raw-output '.digest')" 326 # get second level single manifest 327 submanifestJson="$( 328 curl -fsSL \ 329 -H "Authorization: Bearer $token" \ 330 -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ 331 -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ 332 -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ 333 "$registryBase/v2/$image/manifests/$digest" 334 )" 335 handle_single_manifest_v2 "$submanifestJson" 336 found="found" 337 break 338 fi 339 done 340 if [ -z "$found" ]; then 341 echo >&2 "error: manifest for ${targetArch}${targetVariant:+/${targetVariant}} is not found" 342 exit 1 343 fi 344 ;; 345 *) 346 echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'" 347 exit 1 348 ;; 349 esac 350 ;; 351 352 1) 353 if [ -z "$doNotGenerateManifestJson" ]; then 354 echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'" 355 echo >&2 " this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)" 356 echo >&2 357 doNotGenerateManifestJson=1 358 fi 359 360 layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')" 361 IFS="$newlineIFS" 362 mapfile -t layers <<< "$layersFs" 363 unset IFS 364 365 history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')" 366 imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')" 367 368 echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." 369 for i in "${!layers[@]}"; do 370 imageJson="$(echo "$history" | jq --raw-output ".[${i}]")" 371 layerId="$(echo "$imageJson" | jq --raw-output '.id')" 372 imageLayer="${layers[$i]}" 373 374 mkdir -p "$dir/$layerId" 375 echo '1.0' > "$dir/$layerId/VERSION" 376 377 echo "$imageJson" > "$dir/$layerId/json" 378 379 # TODO figure out why "-C -" doesn't work here 380 # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." 381 # "HTTP/1.1 416 Requested Range Not Satisfiable" 382 if [ -f "$dir/$layerId/layer.tar" ]; then 383 # TODO hackpatch for no -C support :'( 384 echo "skipping existing ${layerId:0:12}" 385 continue 386 fi 387 token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" 388 fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress-bar 389 done 390 ;; 391 392 *) 393 echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'" 394 exit 1 395 ;; 396 esac 397 398 echo 399 400 if [ -s "$dir/tags-$imageFile.tmp" ]; then 401 echo -n ', ' >> "$dir/tags-$imageFile.tmp" 402 else 403 images=("${images[@]}" "$image") 404 fi 405 echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp" 406 done 407 408 echo -n '{' > "$dir/repositories" 409 firstImage=1 410 for image in "${images[@]}"; do 411 imageFile="${image//\//_}" # "/" can't be in filenames :) 412 image="${image#library\/}" 413 414 [ "$firstImage" ] || echo -n ',' >> "$dir/repositories" 415 firstImage= 416 echo -n $'\n\t' >> "$dir/repositories" 417 echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories" 418 done 419 echo -n $'\n}\n' >> "$dir/repositories" 420 421 rm -f "$dir"/tags-*.tmp 422 423 if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then 424 echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json" 425 else 426 rm -f "$dir/manifest.json" 427 fi 428 429 echo "Download of images into '$dir' complete." 430 echo "Use something like the following to load the result into a Docker daemon:" 431 echo " tar -cC '$dir' . | docker load"