From 3ab8a81c14d5ddd9ce7e39b025a132605c252192 Mon Sep 17 00:00:00 2001
From: Steve Cliff <steve@devcloud.guru>
Date: Fri, 21 Aug 2026 17:30:33 +0100
Subject: [PATCH] Retry image pushes on transient registry failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The engine images carry a ~5.6GB torch layer. Uploading it intermittently
fails with a 502 from the reverse proxy in front of the registry, and the
manifest PUT that follows can fail with a 500 because the blob commit has
not registered yet. Both clear on a retry, but the script pushed each tag
exactly once, so a whole release — including a 25GB rebuild — could be lost
to one hiccup.

push_image() replaces the bare `run docker push` calls and retries up to
PUSH_RETRIES times (default 5) with PUSH_RETRY_DELAY seconds between
attempts (default 10). Exhausting the retries still returns non-zero so
set -e aborts the release rather than reporting a partial push as success.
Dry runs make no docker calls.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 DEVELOPER.md      |  9 +++++++++
 release-engine.sh | 46 ++++++++++++++++++++++++++++++++++++++++------
 2 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/DEVELOPER.md b/DEVELOPER.md
index a32c78e..b2b849d 100644
--- a/DEVELOPER.md
+++ b/DEVELOPER.md
@@ -101,6 +101,15 @@ Override the registry and org via environment variables:
 REGISTRY=ghcr.io IMAGE_ORG=myorg ./release-engine.sh --github
 ```
 
+Pushes are retried on transient registry failures. The engine images carry a
+~5.6GB torch layer, and uploading it can fail with a 502 from the proxy in
+front of the registry (or a 500 on the manifest PUT that follows), which
+clears on a retry. Tune with:
+
+```bash
+PUSH_RETRIES=8 PUSH_RETRY_DELAY=20 ./release-engine.sh --gitea
+```
+
 ## API reference
 
 All endpoints are under `/api/v1/`. Requires `Authorization: Bearer <key>` header when `KB_API_KEY` is set.
diff --git a/release-engine.sh b/release-engine.sh
index 24253c7..640de67 100755
--- a/release-engine.sh
+++ b/release-engine.sh
@@ -23,6 +23,10 @@ REGISTRY="${REGISTRY:-docker.dcglab.co.uk}"
 IMAGE_ORG="${IMAGE_ORG:-}"
 IMAGE_BASE="${REGISTRY}${IMAGE_ORG:+/${IMAGE_ORG}}/kb"
 
+# Push retries — see push_image() below
+PUSH_RETRIES="${PUSH_RETRIES:-5}"
+PUSH_RETRY_DELAY="${PUSH_RETRY_DELAY:-10}"
+
 #──────────────────────────────────────────────────────────────────────
 # Parse args
 #──────────────────────────────────────────────────────────────────────
@@ -102,6 +106,36 @@ run() {
     fi
 }
 
+# Push one image tag, retrying on transient registry failures.
+#
+# The engine images carry a ~5.6GB torch layer. Uploading it intermittently
+# fails with a 502 from the reverse proxy in front of the registry, and a
+# manifest PUT can then fail with a 500 because the blob commit has not yet
+# registered. Both clear on a retry, so a whole release should not be lost to
+# one hiccup. Tune with PUSH_RETRIES / PUSH_RETRY_DELAY.
+push_image() {
+    local image="$1"
+    local attempt=1
+
+    echo "  $ docker push $image"
+    [[ "$DRY_RUN" == true ]] && return 0
+
+    while true; do
+        if docker push "$image"; then
+            return 0
+        fi
+
+        if (( attempt >= PUSH_RETRIES )); then
+            echo "Error: failed to push $image after $PUSH_RETRIES attempts" >&2
+            return 1
+        fi
+
+        echo "    push failed (attempt $attempt/$PUSH_RETRIES) — retrying in ${PUSH_RETRY_DELAY}s"
+        sleep "$PUSH_RETRY_DELAY"
+        attempt=$(( attempt + 1 ))
+    done
+}
+
 #──────────────────────────────────────────────────────────────────────
 # Determine release version
 #──────────────────────────────────────────────────────────────────────
@@ -237,14 +271,14 @@ echo ""
 #──────────────────────────────────────────────────────────────────────
 echo "==> Pushing Docker images to $REGISTRY"
 
-run docker push "$NVIDIA_IMAGE"
-run docker push "$NVIDIA_LATEST"
-run docker push "$CPU_IMAGE"
-run docker push "$CPU_LATEST"
+push_image "$NVIDIA_IMAGE"
+push_image "$NVIDIA_LATEST"
+push_image "$CPU_IMAGE"
+push_image "$CPU_LATEST"
 
 if [[ -n "${MCP_IMAGE:-}" ]]; then
-    run docker push "$MCP_IMAGE"
-    run docker push "$MCP_LATEST"
+    push_image "$MCP_IMAGE"
+    push_image "$MCP_LATEST"
 fi
 
 echo ""