Skip to content

Renew the registry token when a layer download fails mid-import #279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
100milliongold wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Renew the registry token when a layer download fails mid-import #279

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 26 additions & 4 deletions src/docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ docker::_download() {
local image="$3"

local req_params=() layers=() layer_media_types=() missing_digests=() missing_media_types=()
local manifest= config= digest= media_type= idx=
local manifest= config= digest= media_type= idx= rv= cached= cached_prev=-1
local accept_manifest_list=("-H" "Accept: application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json")
local accept_manifest=("-H" "Accept: application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json")
local url_manifest="${curl_proto}://${registry}/v2/${image}/manifests/${tag}"
Expand Down Expand Up @@ -185,9 +185,31 @@ docker::_download() {
# Download digests, verify their checksums and extract them in the cache.
if [ "${#missing_digests[@]}" -gt 0 ]; then
common::log INFO "Downloading ${#missing_digests[@]} missing layers..." NL
BASH_ENV="${BASH_SOURCE[0]}" parallel --plain ${TTY_ON+--bar} --xapply --shuf --retries 2 -j "${ENROOT_MAX_CONNECTIONS}" -q \
docker::_download_extract "{1}" "{2}" "${curl_opts[@]}" -f "${req_params[@]}" -- "${url_digest}sha256:{1}" ::: "${missing_digests[@]}" ::: "${missing_media_types[@]}"
common::log
while :; do
rv=0
BASH_ENV="${BASH_SOURCE[0]}" parallel --plain ${TTY_ON+--bar} --xapply --shuf --retries 2 -j "${ENROOT_MAX_CONNECTIONS}" -q \
docker::_download_extract "{1}" "{2}" "${curl_opts[@]}" -f "${req_params[@]}" -- "${url_digest}sha256:{1}" ::: "${missing_digests[@]}" ::: "${missing_media_types[@]}" || rv=$?
common::log
[ "${rv}" -eq 0 ] && break

# The registry token is fetched once above and can expire before the
# last layer is requested, which fails the download with a 401 that
# neither curl nor parallel can retry into a success. Renew it and
# resume: layers already in the cache exit early on the next pass.
# Only resume while the cache keeps growing, so a download that is
# failing for any other reason still stops.
cached=0
for digest in "${layers[@]}"; do
[ -e "${ENROOT_CACHE_PATH}/${digest}" ] && cached=$((cached + 1))
done
if [ "${cached}" -le "${cached_prev}" ]; then
common::err "Could not download all the layers of ${registry}/${image}"
fi
cached_prev="${cached}"

common::log INFO "Download interrupted, renewing the registry token and resuming"
docker::_authenticate "${user}" "${registry}" "${url_manifest}"
done
else
common::log INFO "Found all layers in cache"
fi
Expand Down