diff --git a/DEVELOPER.md b/DEVELOPER.md index a32c78e..b2b849d 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -101,6 +101,15 @@ Override the registry and org via environment variables: REGISTRY=ghcr.io IMAGE_ORG=myorg ./release-engine.sh --github ``` +Pushes are retried on transient registry failures. The engine images carry a +~5.6GB torch layer, and uploading it can fail with a 502 from the proxy in +front of the registry (or a 500 on the manifest PUT that follows), which +clears on a retry. Tune with: + +```bash +PUSH_RETRIES=8 PUSH_RETRY_DELAY=20 ./release-engine.sh --gitea +``` + ## API reference All endpoints are under `/api/v1/`. Requires `Authorization: Bearer ` header when `KB_API_KEY` is set. diff --git a/release-engine.sh b/release-engine.sh index 24253c7..640de67 100755 --- a/release-engine.sh +++ b/release-engine.sh @@ -23,6 +23,10 @@ REGISTRY="${REGISTRY:-docker.dcglab.co.uk}" IMAGE_ORG="${IMAGE_ORG:-}" IMAGE_BASE="${REGISTRY}${IMAGE_ORG:+/${IMAGE_ORG}}/kb" +# Push retries — see push_image() below +PUSH_RETRIES="${PUSH_RETRIES:-5}" +PUSH_RETRY_DELAY="${PUSH_RETRY_DELAY:-10}" + #────────────────────────────────────────────────────────────────────── # Parse args #────────────────────────────────────────────────────────────────────── @@ -102,6 +106,36 @@ run() { fi } +# Push one image tag, retrying on transient registry failures. +# +# The engine images carry a ~5.6GB torch layer. Uploading it intermittently +# fails with a 502 from the reverse proxy in front of the registry, and a +# manifest PUT can then fail with a 500 because the blob commit has not yet +# registered. Both clear on a retry, so a whole release should not be lost to +# one hiccup. Tune with PUSH_RETRIES / PUSH_RETRY_DELAY. +push_image() { + local image="$1" + local attempt=1 + + echo " $ docker push $image" + [[ "$DRY_RUN" == true ]] && return 0 + + while true; do + if docker push "$image"; then + return 0 + fi + + if (( attempt >= PUSH_RETRIES )); then + echo "Error: failed to push $image after $PUSH_RETRIES attempts" >&2 + return 1 + fi + + echo " push failed (attempt $attempt/$PUSH_RETRIES) — retrying in ${PUSH_RETRY_DELAY}s" + sleep "$PUSH_RETRY_DELAY" + attempt=$(( attempt + 1 )) + done +} + #────────────────────────────────────────────────────────────────────── # Determine release version #────────────────────────────────────────────────────────────────────── @@ -237,14 +271,14 @@ echo "" #────────────────────────────────────────────────────────────────────── echo "==> Pushing Docker images to $REGISTRY" -run docker push "$NVIDIA_IMAGE" -run docker push "$NVIDIA_LATEST" -run docker push "$CPU_IMAGE" -run docker push "$CPU_LATEST" +push_image "$NVIDIA_IMAGE" +push_image "$NVIDIA_LATEST" +push_image "$CPU_IMAGE" +push_image "$CPU_LATEST" if [[ -n "${MCP_IMAGE:-}" ]]; then - run docker push "$MCP_IMAGE" - run docker push "$MCP_LATEST" + push_image "$MCP_IMAGE" + push_image "$MCP_LATEST" fi echo ""