github.com/docker/docker@v299999999.0.0-20200612211812-aaf470eca7b5+incompatible/contrib/download-frozen-image-v2.sh (about) 1 #!/usr/bin/env bash 2 set -eo pipefail 3 4 # hello-world latest ef872312fe1b 3 months ago 910 B 5 # hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B 6 7 # debian latest f6fab3b798be 10 weeks ago 85.1 MB 8 # debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB 9 10 # check if essential commands are in our PATH 11 for cmd in curl jq go; do 12 if ! command -v $cmd &> /dev/null; then 13 echo >&2 "error: \"$cmd\" not found!" 14 exit 1 15 fi 16 done 17 18 usage() { 19 echo "usage: $0 dir image[:tag][@digest] ..." 20 echo " $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7" 21 [ -z "$1" ] || exit "$1" 22 } 23 24 dir="$1" # dir for building tar in 25 shift || usage 1 >&2 26 27 if ! [ $# -gt 0 ] && [ "$dir" ]; then 28 usage 2 >&2 29 fi 30 mkdir -p "$dir" 31 32 # hacky workarounds for Bash 3 support (no associative arrays) 33 images=() 34 rm -f "$dir"/tags-*.tmp 35 manifestJsonEntries=() 36 doNotGenerateManifestJson= 37 # repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."' 38 39 # bash v4 on Windows CI requires CRLF separator 40 newlineIFS=$'\n' 41 if [ "$(go env GOHOSTOS)" = 'windows' ]; then 42 major=$(echo "${BASH_VERSION%%[^0.9]}" | cut -d. -f1) 43 if [ "$major" -ge 4 ]; then 44 newlineIFS=$'\r\n' 45 fi 46 fi 47 48 registryBase='https://registry-1.docker.io' 49 authBase='https://auth.docker.io' 50 authService='registry.docker.io' 51 52 # https://github.com/moby/moby/issues/33700 53 fetch_blob() { 54 local token="$1" 55 shift 56 local image="$1" 57 shift 58 local digest="$1" 59 shift 60 local targetFile="$1" 61 shift 62 local curlArgs=("$@") 63 64 local curlHeaders 65 curlHeaders="$( 66 curl -S "${curlArgs[@]}" \ 67 -H "Authorization: Bearer $token" \ 68 "$registryBase/v2/$image/blobs/$digest" \ 69 -o "$targetFile" \ 70 -D- 71 )" 72 curlHeaders="$(echo "$curlHeaders" | tr -d '\r')" 73 if grep -qE "^HTTP/[0-9].[0-9] 3" <<< "$curlHeaders"; then 74 rm -f "$targetFile" 75 76 local blobRedirect 77 blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')" 78 if [ -z "$blobRedirect" ]; then 79 echo >&2 "error: failed fetching '$image' blob '$digest'" 80 echo "$curlHeaders" | head -1 >&2 81 return 1 82 fi 83 84 curl -fSL "${curlArgs[@]}" \ 85 "$blobRedirect" \ 86 -o "$targetFile" 87 fi 88 } 89 90 # handle 'application/vnd.docker.distribution.manifest.v2+json' manifest 91 handle_single_manifest_v2() { 92 local manifestJson="$1" 93 shift 94 95 local configDigest 96 configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')" 97 local imageId="${configDigest#*:}" # strip off "sha256:" 98 99 local configFile="$imageId.json" 100 fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s 101 102 local layersFs 103 layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" 104 local IFS="$newlineIFS" 105 local layers 106 mapfile -t layers <<< "$layersFs" 107 unset IFS 108 109 echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." 110 local layerId= 111 local layerFiles=() 112 for i in "${!layers[@]}"; do 113 local layerMeta="${layers[$i]}" 114 115 local layerMediaType 116 layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')" 117 local layerDigest 118 layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')" 119 120 # save the previous layer's ID 121 local parentId="$layerId" 122 # create a new fake layer ID based on this layer's digest and the previous layer's fake ID 123 layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)" 124 # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value) 125 126 mkdir -p "$dir/$layerId" 127 echo '1.0' > "$dir/$layerId/VERSION" 128 129 if [ ! -s "$dir/$layerId/json" ]; then 130 local parentJson 131 parentJson="$(printf ', parent: "%s"' "$parentId")" 132 local addJson 133 addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")" 134 # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers 135 jq "$addJson + ." > "$dir/$layerId/json" <<- 'EOJSON' 136 { 137 "created": "0001-01-01T00:00:00Z", 138 "container_config": { 139 "Hostname": "", 140 "Domainname": "", 141 "User": "", 142 "AttachStdin": false, 143 "AttachStdout": false, 144 "AttachStderr": false, 145 "Tty": false, 146 "OpenStdin": false, 147 "StdinOnce": false, 148 "Env": null, 149 "Cmd": null, 150 "Image": "", 151 "Volumes": null, 152 "WorkingDir": "", 153 "Entrypoint": null, 154 "OnBuild": null, 155 "Labels": null 156 } 157 } 158 EOJSON 159 fi 160 161 case "$layerMediaType" in 162 application/vnd.docker.image.rootfs.diff.tar.gzip) 163 local layerTar="$layerId/layer.tar" 164 layerFiles=("${layerFiles[@]}" "$layerTar") 165 # TODO figure out why "-C -" doesn't work here 166 # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." 167 # "HTTP/1.1 416 Requested Range Not Satisfiable" 168 if [ -f "$dir/$layerTar" ]; then 169 # TODO hackpatch for no -C support :'( 170 echo "skipping existing ${layerId:0:12}" 171 continue 172 fi 173 local token 174 token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" 175 fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress-bar 176 ;; 177 178 *) 179 echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'" 180 exit 1 181 ;; 182 esac 183 done 184 185 # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons) 186 imageId="$layerId" 187 188 # munge the top layer image manifest to have the appropriate image configuration for older daemons 189 local imageOldConfig 190 imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")" 191 jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json" 192 193 local manifestJsonEntry 194 manifestJsonEntry="$( 195 echo '{}' | jq --raw-output '. + { 196 Config: "'"$configFile"'", 197 RepoTags: ["'"${image#library\/}:$tag"'"], 198 Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"' 199 }' 200 )" 201 manifestJsonEntries=("${manifestJsonEntries[@]}" "$manifestJsonEntry") 202 } 203 204 while [ $# -gt 0 ]; do 205 imageTag="$1" 206 shift 207 image="${imageTag%%[:@]*}" 208 imageTag="${imageTag#*:}" 209 digest="${imageTag##*@}" 210 tag="${imageTag%%@*}" 211 212 # add prefix library if passed official image 213 if [[ "$image" != *"/"* ]]; then 214 image="library/$image" 215 fi 216 217 imageFile="${image//\//_}" # "/" can't be in filenames :) 218 219 token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" 220 221 manifestJson="$( 222 curl -fsSL \ 223 -H "Authorization: Bearer $token" \ 224 -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ 225 -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ 226 -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ 227 "$registryBase/v2/$image/manifests/$digest" 228 )" 229 if [ "${manifestJson:0:1}" != '{' ]; then 230 echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" 231 echo >&2 " $manifestJson" 232 exit 1 233 fi 234 235 imageIdentifier="$image:$tag@$digest" 236 237 schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')" 238 case "$schemaVersion" in 239 2) 240 mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')" 241 242 case "$mediaType" in 243 application/vnd.docker.distribution.manifest.v2+json) 244 handle_single_manifest_v2 "$manifestJson" 245 ;; 246 application/vnd.docker.distribution.manifest.list.v2+json) 247 layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')" 248 IFS="$newlineIFS" 249 mapfile -t layers <<< "$layersFs" 250 unset IFS 251 252 found="" 253 # parse first level multi-arch manifest 254 for i in "${!layers[@]}"; do 255 layerMeta="${layers[$i]}" 256 maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')" 257 if [ "$maniArch" = "$(go env GOARCH)" ]; then 258 digest="$(echo "$layerMeta" | jq --raw-output '.digest')" 259 # get second level single manifest 260 submanifestJson="$( 261 curl -fsSL \ 262 -H "Authorization: Bearer $token" \ 263 -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ 264 -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ 265 -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ 266 "$registryBase/v2/$image/manifests/$digest" 267 )" 268 handle_single_manifest_v2 "$submanifestJson" 269 found="found" 270 break 271 fi 272 done 273 if [ -z "$found" ]; then 274 echo >&2 "error: manifest for $maniArch is not found" 275 exit 1 276 fi 277 ;; 278 *) 279 echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'" 280 exit 1 281 ;; 282 esac 283 ;; 284 285 1) 286 if [ -z "$doNotGenerateManifestJson" ]; then 287 echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'" 288 echo >&2 " this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)" 289 echo >&2 290 doNotGenerateManifestJson=1 291 fi 292 293 layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')" 294 IFS="$newlineIFS" 295 mapfile -t layers <<< "$layersFs" 296 unset IFS 297 298 history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')" 299 imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')" 300 301 echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." 302 for i in "${!layers[@]}"; do 303 imageJson="$(echo "$history" | jq --raw-output ".[${i}]")" 304 layerId="$(echo "$imageJson" | jq --raw-output '.id')" 305 imageLayer="${layers[$i]}" 306 307 mkdir -p "$dir/$layerId" 308 echo '1.0' > "$dir/$layerId/VERSION" 309 310 echo "$imageJson" > "$dir/$layerId/json" 311 312 # TODO figure out why "-C -" doesn't work here 313 # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." 314 # "HTTP/1.1 416 Requested Range Not Satisfiable" 315 if [ -f "$dir/$layerId/layer.tar" ]; then 316 # TODO hackpatch for no -C support :'( 317 echo "skipping existing ${layerId:0:12}" 318 continue 319 fi 320 token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" 321 fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress-bar 322 done 323 ;; 324 325 *) 326 echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'" 327 exit 1 328 ;; 329 esac 330 331 echo 332 333 if [ -s "$dir/tags-$imageFile.tmp" ]; then 334 echo -n ', ' >> "$dir/tags-$imageFile.tmp" 335 else 336 images=("${images[@]}" "$image") 337 fi 338 echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp" 339 done 340 341 echo -n '{' > "$dir/repositories" 342 firstImage=1 343 for image in "${images[@]}"; do 344 imageFile="${image//\//_}" # "/" can't be in filenames :) 345 image="${image#library\/}" 346 347 [ "$firstImage" ] || echo -n ',' >> "$dir/repositories" 348 firstImage= 349 echo -n $'\n\t' >> "$dir/repositories" 350 echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories" 351 done 352 echo -n $'\n}\n' >> "$dir/repositories" 353 354 rm -f "$dir"/tags-*.tmp 355 356 if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then 357 echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json" 358 else 359 rm -f "$dir/manifest.json" 360 fi 361 362 echo "Download of images into '$dir' complete." 363 echo "Use something like the following to load the result into a Docker daemon:" 364 echo " tar -cC '$dir' . | docker load"