From 58b1ac7c0847d8389c2620f399ecbd94811450e3 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 15 Jul 2026 17:01:59 -0400 Subject: [PATCH] Harden registry auth retries across CICD image lanes --- .gitea/workflows/cicd.yaml | 270 +++++++++++++++++++++++++++++++++++-- 1 file changed, 258 insertions(+), 12 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 6c7fc88..7b8a8bf 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -171,6 +171,44 @@ jobs: rm -f "${headers_file}" } + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" @@ -185,7 +223,22 @@ jobs: ensure_registry_auth_realm_host - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + login_ok=false + for i in 1 2 3 4 5; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi if [ "${FORCE_REBUILD:-false}" = "true" ]; then echo "needs_build=true" >> "$GITHUB_OUTPUT" @@ -219,6 +272,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -264,7 +318,7 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" PLAYWRIGHT_BROWSERS_UPSTREAM_TAG="mcr.microsoft.com/playwright:v1.56.1-jammy" @@ -420,6 +474,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -482,7 +537,7 @@ jobs: ensure_registry_auth_realm_host - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")" if ! is_hex "${RESOLVED_HEAD_SHA}"; then @@ -614,7 +669,24 @@ jobs: if docker image inspect "${IMAGE}" >/dev/null 2>&1; then echo "Using cached CICD image: ${IMAGE}" else - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + login_ok=false + for i in 1 2 3 4 5; do + if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi + pulled=false for i in 1 2 3; do echo "Pull attempt ${i}/3 for ${IMAGE}" @@ -857,6 +929,24 @@ jobs: run: | set -euo pipefail + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -867,6 +957,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -887,7 +978,7 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base" BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}" BACKEND_BASE_CACHE_REF="${BACKEND_BASE_REPO}:cache" @@ -961,6 +1052,10 @@ jobs: return 1 } + docker_login_with_retry() { + retry_cmd "${1:-5}" "${2:-3}" sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin' + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -971,6 +1066,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -991,7 +1087,7 @@ jobs: return 1 } - retry_cmd 5 3 sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin' + docker_login_with_retry 5 3 FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base" FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}" FRONTEND_BASE_CACHE_REF="${FRONTEND_BASE_REPO}:cache" @@ -1041,6 +1137,46 @@ jobs: run: | set -euo pipefail + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1051,6 +1187,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1071,7 +1208,7 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration" INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}" INTEGRATION_TESTER_CACHE_REF="${INTEGRATION_TESTER_REPO}:cache" @@ -1124,6 +1261,24 @@ jobs: PLAYWRIGHT_BASE_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" PLAYWRIGHT_BASE_IMAGE_UPSTREAM="mcr.microsoft.com/playwright:v1.56.1-jammy" + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_base_pull() { image_ref="$1" attempts="${2:-6}" @@ -1166,6 +1321,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1186,7 +1342,7 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" E2E_TESTER_CACHE_REF="${E2E_TESTER_REPO}:cache" @@ -1246,6 +1402,24 @@ jobs: run: | set -euo pipefail + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1256,6 +1430,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1280,6 +1455,7 @@ jobs: DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" + docker_login_with_retry 5 3 retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \ @@ -1287,7 +1463,6 @@ jobs: -t deployable-backend:"${HEAD_SHA}" . bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}" @@ -1329,6 +1504,46 @@ jobs: run: | set -euo pipefail + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1339,6 +1554,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1363,6 +1579,7 @@ jobs: DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" + docker_login_with_retry 5 3 retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ --target production \ @@ -1371,7 +1588,6 @@ jobs: -t deployable-frontend:"${HEAD_SHA}" . bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}" @@ -1655,7 +1871,22 @@ jobs: exit 1 fi - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + login_ok=false + for i in 1 2 3 4 5; do + if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")" @@ -1949,7 +2180,22 @@ jobs: exit 1 fi - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + login_ok=false + for i in 1 2 3 4 5; do + if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"