Harden registry auth retries across CICD image lanes

.gitea/workflows/cicd.yaml

@@ -171,6 +171,44 @@ jobs:
             rm -f "${headers_file}"
           }
 
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              ensure_registry_auth_realm_host
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              ensure_registry_auth_realm_host
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
           BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
           BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
           BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest"
@@ -185,7 +223,22 @@ jobs:
 
           ensure_registry_auth_realm_host
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null
+          login_ok=false
+          for i in 1 2 3 4 5; do
+            if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null; then
+              login_ok=true
+              break
+            fi
+            if [ "${i}" -lt 5 ]; then
+              sleep_seconds=$((3 * i))
+              echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
+              sleep "${sleep_seconds}"
+            fi
+          done
+          if [ "${login_ok}" != "true" ]; then
+            echo "❌ Failed docker login after retries"
+            exit 1
+          fi
 
           if [ "${FORCE_REBUILD:-false}" = "true" ]; then
             echo "needs_build=true" >> "$GITHUB_OUTPUT"
@@ -219,6 +272,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -264,7 +318,7 @@ jobs:
             echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
           fi
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          docker_login_with_retry 5 3
 
           PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
           PLAYWRIGHT_BROWSERS_UPSTREAM_TAG="mcr.microsoft.com/playwright:v1.56.1-jammy"
@@ -420,6 +474,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -482,7 +537,7 @@ jobs:
 
           ensure_registry_auth_realm_host
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          docker_login_with_retry 5 3
 
           RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")"
           if ! is_hex "${RESOLVED_HEAD_SHA}"; then
@@ -614,7 +669,24 @@ jobs:
           if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
             echo "Using cached CICD image: ${IMAGE}"
           else
-            echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
+            login_ok=false
+            for i in 1 2 3 4 5; do
+              if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
+                login_ok=true
+                break
+              fi
+              if [ "${i}" -lt 5 ]; then
+                sleep_seconds=$((3 * i))
+                echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+            done
+
+            if [ "${login_ok}" != "true" ]; then
+              echo "❌ Failed docker login after retries"
+              exit 1
+            fi
+
             pulled=false
             for i in 1 2 3; do
               echo "Pull attempt ${i}/3 for ${IMAGE}"
@@ -857,6 +929,24 @@ jobs:
         run: |
           set -euo pipefail
 
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
           retry_registry_op() {
             op_name="$1"
             image_ref="$2"
@@ -867,6 +957,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -887,7 +978,7 @@ jobs:
             return 1
           }
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          docker_login_with_retry 5 3
           BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base"
           BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}"
           BACKEND_BASE_CACHE_REF="${BACKEND_BASE_REPO}:cache"
@@ -961,6 +1052,10 @@ jobs:
             return 1
           }
 
+          docker_login_with_retry() {
+            retry_cmd "${1:-5}" "${2:-3}" sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin'
+          }
+
           retry_registry_op() {
             op_name="$1"
             image_ref="$2"
@@ -971,6 +1066,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -991,7 +1087,7 @@ jobs:
             return 1
           }
 
-          retry_cmd 5 3 sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin'
+          docker_login_with_retry 5 3
           FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base"
           FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}"
           FRONTEND_BASE_CACHE_REF="${FRONTEND_BASE_REPO}:cache"
@@ -1041,6 +1137,46 @@ jobs:
         run: |
           set -euo pipefail
 
+          ensure_registry_auth_realm_host() {
+            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
+              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
+            fi
+
+            headers_file="$(mktemp)"
+            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
+              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
+              if [ -n "${realm_url}" ]; then
+                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
+                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
+                  if ! grep -q "${realm_host}" /etc/hosts; then
+                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
+                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
+                  fi
+                fi
+              fi
+            fi
+            rm -f "${headers_file}"
+          }
+
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              ensure_registry_auth_realm_host
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
           retry_registry_op() {
             op_name="$1"
             image_ref="$2"
@@ -1051,6 +1187,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -1071,7 +1208,7 @@ jobs:
             return 1
           }
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          docker_login_with_retry 5 3
           INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration"
           INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}"
           INTEGRATION_TESTER_CACHE_REF="${INTEGRATION_TESTER_REPO}:cache"
@@ -1124,6 +1261,24 @@ jobs:
           PLAYWRIGHT_BASE_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
           PLAYWRIGHT_BASE_IMAGE_UPSTREAM="mcr.microsoft.com/playwright:v1.56.1-jammy"
 
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
           retry_base_pull() {
             image_ref="$1"
             attempts="${2:-6}"
@@ -1166,6 +1321,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -1186,7 +1342,7 @@ jobs:
             return 1
           }
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          docker_login_with_retry 5 3
           E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e"
           E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}"
           E2E_TESTER_CACHE_REF="${E2E_TESTER_REPO}:cache"
@@ -1246,6 +1402,24 @@ jobs:
         run: |
           set -euo pipefail
 
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
           retry_registry_op() {
             op_name="$1"
             image_ref="$2"
@@ -1256,6 +1430,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -1280,6 +1455,7 @@ jobs:
           DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
           DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache"
 
+          docker_login_with_retry 5 3
           retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true
           DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
             --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \
@@ -1287,7 +1463,6 @@ jobs:
             -t deployable-backend:"${HEAD_SHA}" .
           bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
           docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
           docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}"
 
@@ -1329,6 +1504,46 @@ jobs:
         run: |
           set -euo pipefail
 
+          ensure_registry_auth_realm_host() {
+            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
+              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
+            fi
+
+            headers_file="$(mktemp)"
+            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
+              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
+              if [ -n "${realm_url}" ]; then
+                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
+                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
+                  if ! grep -q "${realm_host}" /etc/hosts; then
+                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
+                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
+                  fi
+                fi
+              fi
+            fi
+            rm -f "${headers_file}"
+          }
+
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              ensure_registry_auth_realm_host
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
           retry_registry_op() {
             op_name="$1"
             image_ref="$2"
@@ -1339,6 +1554,7 @@ jobs:
             while [ "${attempt}" -le "${attempts}" ]; do
               echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
               if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
                 if docker push "${image_ref}"; then
                   return 0
                 fi
@@ -1363,6 +1579,7 @@ jobs:
           DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
           DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache"
 
+          docker_login_with_retry 5 3
           retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true
           BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
             --target production \
@@ -1371,7 +1588,6 @@ jobs:
             -t deployable-frontend:"${HEAD_SHA}" .
           bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend
 
-          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
           docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
           docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}"
 
@@ -1655,7 +1871,22 @@ jobs:
             exit 1
           fi
 
-          echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
+          login_ok=false
+          for i in 1 2 3 4 5; do
+            if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
+              login_ok=true
+              break
+            fi
+            if [ "${i}" -lt 5 ]; then
+              sleep_seconds=$((3 * i))
+              echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
+              sleep "${sleep_seconds}"
+            fi
+          done
+          if [ "${login_ok}" != "true" ]; then
+            echo "❌ Failed docker login after retries"
+            exit 1
+          fi
 
           RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
           RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"
@@ -1949,7 +2180,22 @@ jobs:
             exit 1
           fi
 
-          echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
+          login_ok=false
+          for i in 1 2 3 4 5; do
+            if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
+              login_ok=true
+              break
+            fi
+            if [ "${i}" -lt 5 ]; then
+              sleep_seconds=$((3 * i))
+              echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
+              sleep "${sleep_seconds}"
+            fi
+          done
+          if [ "${login_ok}" != "true" ]; then
+            echo "❌ Failed docker login after retries"
+            exit 1
+          fi
 
           RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
           RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"