diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 545730f..43f74c4 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2,6 +2,7 @@ name: CICD on: push: + branches: [main, develop] pull_request: branches: [main, develop] workflow_dispatch: @@ -9,16 +10,10 @@ on: head_sha: description: Commit SHA to process required: false - base_hash: - description: Immutable base hash to use for CICD base image - required: false force_rebuild_base: description: Force CICD base rebuild required: false default: "false" - trace_id: - description: Correlation id propagated across CICD jobs - required: false env: GITEA_SSH_HOST: kankali.darkhelm.lan @@ -138,12 +133,13 @@ jobs: if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh else - git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh fi chmod +x scripts/compute-cicd-base-hash.sh + chmod +x scripts/resolve-mirrored-image.sh - name: Compute base hash and inspect registry id: base-state @@ -153,6 +149,66 @@ jobs: FORCE_REBUILD: ${{ github.event.inputs.force_rebuild_base }} run: | set -euo pipefail + + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" @@ -165,7 +221,24 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + ensure_registry_auth_realm_host + + login_ok=false + for i in 1 2 3 4 5; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi if [ "${FORCE_REBUILD:-false}" = "true" ]; then echo "needs_build=true" >> "$GITHUB_OUTPUT" @@ -199,6 +272,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -240,16 +314,45 @@ jobs: rm -f "${headers_file}" } + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" - retry_registry_op pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 5 3 + PLAYWRIGHT_BROWSERS_UPSTREAM_TAG="mcr.microsoft.com/playwright:v1.56.1-jammy" + PLAYWRIGHT_BROWSERS_LOCAL_TAG="$(./scripts/resolve-mirrored-image.sh \ + --image "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" \ + --mirror-image "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" \ + --upstream-image "${PLAYWRIGHT_BROWSERS_UPSTREAM_TAG}" \ + --registry-user "${REGISTRY_USER}" \ + --registry-password-env PACKAGE_ACCESS_TOKEN)" - PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}") + PLAYWRIGHT_BROWSERS_IMAGE="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${PLAYWRIGHT_BROWSERS_LOCAL_TAG}" | head -n 1; } || true)" + if [ -z "${PLAYWRIGHT_BROWSERS_IMAGE}" ]; then + echo "❌ Unable to resolve Playwright browsers digest reference" + exit 1 + fi docker build --progress=plain -f Dockerfile.cicd-base \ --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \ @@ -346,11 +449,12 @@ jobs: if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh + git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh else - git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh + git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh fi chmod +x scripts/compute-cicd-base-hash.sh + chmod +x scripts/resolve-mirrored-image.sh - name: Build and push complete CICD image env: @@ -358,7 +462,6 @@ jobs: REGISTRY_USER: ${{ github.actor }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }} run: | set -euo pipefail @@ -390,6 +493,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -431,12 +535,39 @@ jobs: rm -f "${headers_file}" } + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + echo "=== Pre-build disk telemetry ===" df -h || true if command -v docker >/dev/null 2>&1; then docker system df || true - docker builder prune -af || true - docker system prune -af --volumes || true + used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" + echo "disk_used_before=${used_before}%" + echo "prune_threshold=${threshold}%" + if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then + docker builder prune -af || true + docker system prune -af --volumes || true + else + echo "Skipping prune; usage below threshold" + fi echo "=== Post-prune disk telemetry ===" df -h || true docker system df || true @@ -444,7 +575,7 @@ jobs: ensure_registry_auth_realm_host - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")" if ! is_hex "${RESOLVED_HEAD_SHA}"; then @@ -452,13 +583,13 @@ jobs: exit 1 fi - BASE_HASH="$(trim_spaces "${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}")" + BASE_HASH="$(trim_spaces "${BASE_HASH_FROM_BUILD}")" if ! is_hex "${BASE_HASH}"; then BASE_HASH="$(./scripts/compute-cicd-base-hash.sh | tr -d '[:space:]')" fi if ! is_hex "${BASE_HASH}"; then - echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}'" + echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_FROM_BUILD}'" exit 1 fi @@ -466,21 +597,24 @@ jobs: echo "resolved_base_hash=${BASE_HASH}" BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" + CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" + CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" + retry_registry_op pull "${BASE_IMAGE}" 5 3 + retry_registry_op pull "${CICD_LATEST_REF}" 5 3 || true echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key chmod 600 /tmp/ssh_key - docker build -f Dockerfile.cicd \ + DOCKER_BUILDKIT=1 docker build -f Dockerfile.cicd \ + --cache-from "${CICD_LATEST_REF}" \ --secret id=ssh_private_key,src=/tmp/ssh_key \ --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg GITHUB_SHA="${RESOLVED_HEAD_SHA}" \ --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ -t cicd:latest . - CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" - docker tag cicd:latest "${CICD_LATEST_REF}" docker tag cicd:latest "${CICD_SHA_REF}" retry_registry_op push "${CICD_LATEST_REF}" 5 4 @@ -511,10 +645,10 @@ jobs: fi timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true - backend-tests: - name: Backend Tests + source-checks: + name: Source Checks runs-on: ubuntu-act - timeout-minutes: 25 + timeout-minutes: 75 needs: build_cicd steps: - *identify_runner_step @@ -522,6 +656,27 @@ jobs: - &configure_registry_host_step name: Configure registry host resolution run: | + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + if [ "${CI_PRUNE_AT_JOB_START:-false}" = "true" ] && command -v docker >/dev/null 2>&1; then used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" @@ -541,6 +696,8 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi + ensure_registry_auth_realm_host + - &ensure_cicd_image_step name: Ensure CICD image is available env: @@ -550,7 +707,24 @@ jobs: if docker image inspect "${IMAGE}" >/dev/null 2>&1; then echo "Using cached CICD image: ${IMAGE}" else - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + login_ok=false + for i in 1 2 3 4 5; do + if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi + pulled=false for i in 1 2 3; do echo "Pull attempt ${i}/3 for ${IMAGE}" @@ -592,19 +766,6 @@ jobs: exit "${TEST_STATUS}" fi - - *failure_diagnostics_step - - precommit-tests: - name: Pre-commit Checks - # Source-level checks should be able to use the full ubuntu-act pool. - runs-on: ubuntu-act - timeout-minutes: 30 - needs: build_cicd - steps: - - *identify_runner_step - - - *configure_registry_host_step - - *ensure_cicd_image_step - name: Run pre-commit checks env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} @@ -628,19 +789,6 @@ jobs: exit "${TEST_STATUS}" fi - - *failure_diagnostics_step - - frontend-tests: - name: Frontend Tests - # Source-level checks should be able to use the full ubuntu-act pool. - runs-on: ubuntu-act - timeout-minutes: 25 - needs: build_cicd - steps: - - *identify_runner_step - - - *configure_registry_host_step - - *ensure_cicd_image_step - name: Run frontend tests with coverage env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} @@ -661,19 +809,6 @@ jobs: exit "${TEST_STATUS}" fi - - *failure_diagnostics_step - - xdoctest: - name: Backend Doctests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 15 - needs: build_cicd - steps: - - *identify_runner_step - - - *configure_registry_host_step - - *ensure_cicd_image_step - name: Run backend doctests env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} @@ -683,8 +818,7 @@ jobs: docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " cd /workspace/backend && - source .venv/bin/activate && - uv run xdoctest src/ --quiet + .venv/bin/python -m xdoctest src/ --quiet " 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} @@ -701,6 +835,7 @@ jobs: name: Frontend Dependency Audit runs-on: ubuntu-act timeout-minutes: 15 + continue-on-error: true needs: build_cicd steps: - *identify_runner_step @@ -721,10 +856,10 @@ jobs: TEST_STATUS=${PIPESTATUS[0]} if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Frontend dependency audit failed (exit=${TEST_STATUS})" + echo "⚠️ Frontend dependency audit failed (exit=${TEST_STATUS})" echo "--- Last 200 lines of frontend audit output ---" tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" + echo "Proceeding: frontend audit is informational-only." fi - *failure_diagnostics_step @@ -733,6 +868,7 @@ jobs: name: Backend Dependency Audit runs-on: ubuntu-act timeout-minutes: 15 + continue-on-error: true needs: build_cicd steps: - *identify_runner_step @@ -753,10 +889,10 @@ jobs: TEST_STATUS=${PIPESTATUS[0]} if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Backend dependency audit failed (exit=${TEST_STATUS})" + echo "⚠️ Backend dependency audit failed (exit=${TEST_STATUS})" echo "--- Last 200 lines of backend audit output ---" tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" + echo "Proceeding: backend audit is informational-only." fi - *failure_diagnostics_step @@ -764,31 +900,29 @@ jobs: cicd-tests-complete: name: CICD Tests Complete runs-on: ubuntu-act - needs: [backend-tests, precommit-tests, frontend-tests, xdoctest, frontend-audit, backend-audit] + needs: [source-checks, frontend-audit, backend-audit] if: always() steps: - name: Confirm all source lanes passed run: | set -euo pipefail - backend_status="${{ needs['backend-tests'].result }}" - precommit_status="${{ needs['precommit-tests'].result }}" - frontend_status="${{ needs['frontend-tests'].result }}" - xdoctest_status="${{ needs.xdoctest.result }}" + source_checks_status="${{ needs['source-checks'].result }}" frontend_audit_status="${{ needs['frontend-audit'].result }}" backend_audit_status="${{ needs['backend-audit'].result }}" - echo "backend-tests=${backend_status}" - echo "precommit-tests=${precommit_status}" - echo "frontend-tests=${frontend_status}" - echo "xdoctest=${xdoctest_status}" + echo "source-checks=${source_checks_status}" echo "frontend-audit=${frontend_audit_status}" echo "backend-audit=${backend_audit_status}" - if [ "${backend_status}" != "success" ] || [ "${precommit_status}" != "success" ] || [ "${frontend_status}" != "success" ] || [ "${xdoctest_status}" != "success" ] || [ "${frontend_audit_status}" != "success" ] || [ "${backend_audit_status}" != "success" ]; then + if [ "${source_checks_status}" != "success" ]; then echo "❌ One or more CICD source lanes failed" exit 1 fi + if [ "${frontend_audit_status}" != "success" ] || [ "${backend_audit_status}" != "success" ]; then + echo "⚠️ Dependency audit lane reported non-success (informational only)" + fi + echo "✅ Source checks complete" - *failure_diagnostics_step @@ -796,7 +930,7 @@ jobs: build-backend-base-image: name: Build Backend Base Image runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete] + needs: [build_cicd, source-checks] outputs: backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }} backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }} @@ -807,57 +941,22 @@ jobs: run: | echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" - - name: Minimal checkout for backend base image inputs + - *configure_registry_host_step + - *ensure_cicd_image_step + - &hydrate_build_context_from_cicd_step + name: Populate build context from CICD image env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} run: | set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 + SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + SOURCE_CONTAINER="$(docker create "${SOURCE_IMAGE}")" + cleanup() { + docker rm -f "${SOURCE_CONTAINER}" >/dev/null 2>&1 || true } + trap cleanup EXIT - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock - else - git checkout HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock - fi + docker cp "${SOURCE_CONTAINER}:/workspace/." . - name: Build and push backend base image id: backend_base_ref @@ -868,6 +967,24 @@ jobs: run: | set -euo pipefail + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -878,6 +995,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -898,13 +1016,21 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base" BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}" + BACKEND_BASE_CACHE_REF="${BACKEND_BASE_REPO}:cache" - docker build -f Dockerfile.backend --target dependencies -t plex-playlist-backend-base:"${HEAD_SHA}" . + retry_registry_op pull "${BACKEND_BASE_CACHE_REF}" 5 3 || true + DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ + --target dependencies \ + --cache-from "${BACKEND_BASE_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t plex-playlist-backend-base:"${HEAD_SHA}" . docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}" + docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_CACHE_REF}" retry_registry_op push "${BACKEND_BASE_TAG_REF}" 5 4 + retry_registry_op push "${BACKEND_BASE_CACHE_REF}" 5 4 retry_registry_op pull "${BACKEND_BASE_TAG_REF}" 5 3 BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)" @@ -921,64 +1047,15 @@ jobs: build-frontend-base-image: name: Build Frontend Base Image runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete] + needs: [build_cicd, source-checks] outputs: frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} steps: - *resolve_head_sha_from_build_step - - - name: Minimal checkout for frontend base image inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 - } - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml - else - git checkout HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml - fi + - *configure_registry_host_step + - *ensure_cicd_image_step + - *hydrate_build_context_from_cicd_step - name: Build and push frontend base image id: frontend_base_ref @@ -1013,6 +1090,10 @@ jobs: return 1 } + docker_login_with_retry() { + retry_cmd "${1:-5}" "${2:-3}" sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin' + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1023,6 +1104,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1043,13 +1125,21 @@ jobs: return 1 } - retry_cmd 5 3 sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin' + docker_login_with_retry 5 3 FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base" FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}" + FRONTEND_BASE_CACHE_REF="${FRONTEND_BASE_REPO}:cache" - docker build -f Dockerfile.frontend --target deps -t plex-playlist-frontend-base:"${HEAD_SHA}" . + retry_registry_op pull "${FRONTEND_BASE_CACHE_REF}" 5 3 || true + BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ + --target deps \ + --cache-from "${FRONTEND_BASE_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t plex-playlist-frontend-base:"${HEAD_SHA}" . docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}" + docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_CACHE_REF}" retry_registry_op push "${FRONTEND_BASE_TAG_REF}" 5 4 + retry_registry_op push "${FRONTEND_BASE_CACHE_REF}" 5 4 retry_registry_op pull "${FRONTEND_BASE_TAG_REF}" 5 3 FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)" @@ -1066,64 +1156,15 @@ jobs: build-integration-tester-image: name: Build Integration Tester Image runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete] + needs: [build_cicd, source-checks] outputs: integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} steps: - *resolve_head_sha_from_build_step - - - name: Minimal checkout for integration tester image inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 - } - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.integration-tester - else - git checkout HEAD -- Dockerfile.integration-tester - fi + - *configure_registry_host_step + - *ensure_cicd_image_step + - *hydrate_build_context_from_cicd_step - name: Build and push integration tester image id: integration_tester_ref @@ -1134,6 +1175,46 @@ jobs: run: | set -euo pipefail + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1144,6 +1225,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1164,14 +1246,21 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration" INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}" + INTEGRATION_TESTER_CACHE_REF="${INTEGRATION_TESTER_REPO}:cache" - docker build -f Dockerfile.integration-tester -t plex-playlist-integration:"${HEAD_SHA}" . + retry_registry_op pull "${INTEGRATION_TESTER_CACHE_REF}" 5 3 || true + DOCKER_BUILDKIT=1 docker build -f Dockerfile.integration-tester \ + --cache-from "${INTEGRATION_TESTER_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t plex-playlist-integration:"${HEAD_SHA}" . docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}" + docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_CACHE_REF}" retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4 + retry_registry_op push "${INTEGRATION_TESTER_CACHE_REF}" 5 4 retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3 INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)" @@ -1188,64 +1277,15 @@ jobs: build-e2e-tester-image: name: Build E2E Tester Image runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete] + needs: [build_cicd, source-checks] outputs: e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} steps: - *resolve_head_sha_from_build_step - - - name: Minimal checkout for E2E tester image inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 - } - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.e2e-tester - else - git checkout HEAD -- Dockerfile.e2e-tester - fi + - *configure_registry_host_step + - *ensure_cicd_image_step + - *hydrate_build_context_from_cicd_step - name: Build and push E2E tester image id: e2e_tester_ref @@ -1256,7 +1296,26 @@ jobs: run: | set -euo pipefail - PLAYWRIGHT_BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" + PLAYWRIGHT_BASE_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" + PLAYWRIGHT_BASE_IMAGE_UPSTREAM="mcr.microsoft.com/playwright:v1.56.1-jammy" + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } retry_base_pull() { image_ref="$1" @@ -1300,6 +1359,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1320,17 +1380,30 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + docker_login_with_retry 5 3 E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" + E2E_TESTER_CACHE_REF="${E2E_TESTER_REPO}:cache" + + PLAYWRIGHT_BASE_IMAGE="$(./scripts/resolve-mirrored-image.sh \ + --image "${PLAYWRIGHT_BASE_IMAGE_MIRROR}" \ + --mirror-image "${PLAYWRIGHT_BASE_IMAGE_MIRROR}" \ + --upstream-image "${PLAYWRIGHT_BASE_IMAGE_UPSTREAM}" \ + --registry-user "${REGISTRY_USER}" \ + --registry-password-env PACKAGE_ACCESS_TOKEN)" retry_base_pull "${PLAYWRIGHT_BASE_IMAGE}" 6 5 - DOCKER_BUILDKIT=0 docker build --pull=false -f Dockerfile.e2e-tester \ + retry_registry_op pull "${E2E_TESTER_CACHE_REF}" 5 3 || true + DOCKER_BUILDKIT=1 docker build --pull=false -f Dockerfile.e2e-tester \ + --cache-from "${E2E_TESTER_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg PLAYWRIGHT_BASE_IMAGE="${PLAYWRIGHT_BASE_IMAGE}" \ -t plex-playlist-e2e:"${HEAD_SHA}" . docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}" + docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_CACHE_REF}" retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4 + retry_registry_op push "${E2E_TESTER_CACHE_REF}" 5 4 retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3 E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)" @@ -1347,74 +1420,16 @@ jobs: build-backend-main-image: name: Build Backend Main Image runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete, build-backend-base-image] + needs: [build_cicd, source-checks, build-backend-base-image] outputs: head_sha: ${{ steps.meta.outputs.head_sha }} deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} steps: - *resolve_head_sha_from_build_step - - - name: Minimal checkout for backend main image inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 - } - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- \ - .dockerignore \ - Dockerfile.backend \ - backend \ - scripts/verify-deployable-image-purity.sh - else - git checkout HEAD -- \ - .dockerignore \ - Dockerfile.backend \ - backend \ - scripts/verify-deployable-image-purity.sh - fi + - *configure_registry_host_step + - *ensure_cicd_image_step + - *hydrate_build_context_from_cicd_step - name: Build and push backend main image id: deployable_backend_ref @@ -1425,6 +1440,24 @@ jobs: run: | set -euo pipefail + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1435,6 +1468,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1455,15 +1489,23 @@ jobs: return 1 } - docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" . - bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" + + docker_login_with_retry 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true + DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ + --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t deployable-backend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}" retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 + retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4 retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" @@ -1480,73 +1522,16 @@ jobs: build-frontend-main-image: name: Build Frontend Main Image runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete, build-frontend-base-image] + needs: [build_cicd, source-checks, build-frontend-base-image] outputs: head_sha: ${{ steps.meta.outputs.head_sha }} deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} steps: - *resolve_head_sha_from_build_step - - - name: Minimal checkout for frontend main image inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 - } - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- \ - .dockerignore \ - Dockerfile.frontend \ - frontend \ - scripts/verify-deployable-image-purity.sh - else - git checkout HEAD -- \ - .dockerignore \ - Dockerfile.frontend \ - frontend \ - scripts/verify-deployable-image-purity.sh - fi + - *configure_registry_host_step + - *ensure_cicd_image_step + - *hydrate_build_context_from_cicd_step - name: Build and push frontend main image id: deployable_frontend_ref @@ -1557,6 +1542,46 @@ jobs: run: | set -euo pipefail + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -1567,6 +1592,7 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 if docker push "${image_ref}"; then return 0 fi @@ -1587,15 +1613,24 @@ jobs: return 1 } - docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" . - bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" + DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" + + docker_login_with_retry 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true + BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ + --target production \ + --cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t deployable-frontend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}" retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 + retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4 retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" @@ -1748,8 +1783,8 @@ jobs: # Capture follow-up diagnostics when any source lane dies during platform # setup before step logging is available. runs-on: ubuntu-act - needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] - if: always() && (needs['backend-tests'].result == 'failure' || needs['precommit-tests'].result == 'failure' || needs['frontend-tests'].result == 'failure' || needs.xdoctest.result == 'failure') + needs: [source-checks] + if: always() && needs['source-checks'].result == 'failure' timeout-minutes: 10 steps: - *identify_runner_step @@ -1767,10 +1802,7 @@ jobs: echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" echo "repository=${GITHUB_REPOSITORY:-unknown}" echo "server_url=${GITHUB_SERVER_URL:-unknown}" - echo "backend_tests_result=${{ needs['backend-tests'].result }}" - echo "precommit_tests_result=${{ needs['precommit-tests'].result }}" - echo "frontend_tests_result=${{ needs['frontend-tests'].result }}" - echo "xdoctest_result=${{ needs.xdoctest.result }}" + echo "source_checks_result=${{ needs['source-checks'].result }}" echo "=== Local Runner Telemetry (postmortem job host) ===" uname -a || true @@ -1818,10 +1850,7 @@ jobs: ' return "unknown"' \ '' \ 'targets = {' \ - ' "Backend Tests",' \ - ' "Pre-commit Checks",' \ - ' "Frontend Tests",' \ - ' "Backend Doctests",' \ + ' "Source Checks",' \ '}' \ '' \ 'for job in jobs:' \ @@ -1867,6 +1896,9 @@ jobs: COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}" DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" + POSTGRES_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/postgres:16-alpine" + POSTGRES_IMAGE_UPSTREAM="docker.io/library/postgres:16-alpine" + RESOLVER_PATH="/tmp/resolve-mirrored-image.sh" if ! command -v docker >/dev/null 2>&1; then echo "❌ Docker is required" @@ -1877,10 +1909,40 @@ jobs: exit 1 fi + login_ok=false + for i in 1 2 3 4 5; do + if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi + + RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")" + docker cp "${RESOLVER_CONTAINER}:/workspace/scripts/resolve-mirrored-image.sh" "${RESOLVER_PATH}" + docker rm -f "${RESOLVER_CONTAINER}" >/dev/null 2>&1 || true + chmod +x "${RESOLVER_PATH}" + + POSTGRES_IMAGE="$(${RESOLVER_PATH} \ + --image "${POSTGRES_IMAGE_MIRROR}" \ + --mirror-image "${POSTGRES_IMAGE_MIRROR}" \ + --upstream-image "${POSTGRES_IMAGE_UPSTREAM}" \ + --registry-user "${{ github.actor }}" \ + --registry-password-env PACKAGE_ACCESS_TOKEN)" + cat >"${COMPOSE_FILE}" <<'EOF' services: database: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE} environment: POSTGRES_DB: plex_playlist POSTGRES_USER: plex_user @@ -1987,8 +2049,6 @@ jobs: exit 1 fi - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 @@ -2015,7 +2075,7 @@ jobs: echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" - export DB_PASSWORD DB_URL DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}" + export DB_PASSWORD DB_URL POSTGRES_IMAGE DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}" docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend @@ -2127,7 +2187,6 @@ jobs: - name: Run E2E tests against runtime services via compose env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }} @@ -2146,6 +2205,9 @@ jobs: DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" E2E_REPO_DIR="$(mktemp -d)" + POSTGRES_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/postgres:16-alpine" + POSTGRES_IMAGE_UPSTREAM="docker.io/library/postgres:16-alpine" + RESOLVER_PATH="/tmp/resolve-mirrored-image.sh" if ! command -v docker >/dev/null 2>&1; then echo "❌ Docker is required" @@ -2156,10 +2218,40 @@ jobs: exit 1 fi + login_ok=false + for i in 1 2 3 4 5; do + if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then + login_ok=true + break + fi + if [ "${i}" -lt 5 ]; then + sleep_seconds=$((3 * i)) + echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + if [ "${login_ok}" != "true" ]; then + echo "❌ Failed docker login after retries" + exit 1 + fi + + RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")" + docker cp "${RESOLVER_CONTAINER}:/workspace/scripts/resolve-mirrored-image.sh" "${RESOLVER_PATH}" + docker rm -f "${RESOLVER_CONTAINER}" >/dev/null 2>&1 || true + chmod +x "${RESOLVER_PATH}" + + POSTGRES_IMAGE="$(${RESOLVER_PATH} \ + --image "${POSTGRES_IMAGE_MIRROR}" \ + --mirror-image "${POSTGRES_IMAGE_MIRROR}" \ + --upstream-image "${POSTGRES_IMAGE_UPSTREAM}" \ + --registry-user "${{ github.actor }}" \ + --registry-password-env PACKAGE_ACCESS_TOKEN)" + cat >"${COMPOSE_FILE}" <<'EOF' services: database: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE} environment: POSTGRES_DB: plex_playlist POSTGRES_USER: plex_user @@ -2311,20 +2403,10 @@ jobs: exit 1 fi - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" "${E2E_REPO_DIR}" - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git -C "${E2E_REPO_DIR}" fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git -C "${E2E_REPO_DIR}" checkout FETCH_HEAD -- frontend - else - git -C "${E2E_REPO_DIR}" checkout HEAD -- frontend - fi + SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + SOURCE_CONTAINER="$(docker create "${SOURCE_IMAGE}")" + docker cp "${SOURCE_CONTAINER}:/workspace/frontend" "${E2E_REPO_DIR}/frontend" + docker rm -f "${SOURCE_CONTAINER}" >/dev/null 2>&1 || true if [ ! -f "${E2E_REPO_DIR}/frontend/package.json" ]; then echo "❌ Missing frontend/package.json in cloned E2E workspace" @@ -2332,7 +2414,6 @@ jobs: exit 1 fi - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 @@ -2381,7 +2462,7 @@ jobs: echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}" echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}" - export DB_PASSWORD DB_URL E2E_ARTIFACT_DIR="${ARTIFACT_DIR}" + export DB_PASSWORD DB_URL POSTGRES_IMAGE E2E_ARTIFACT_DIR="${ARTIFACT_DIR}" export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}" export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}" diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 2903806..014db65 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -26,6 +26,8 @@ jobs: RENOVATE_IMAGE_FALLBACK: ghcr.io/renovatebot/renovate:41 GITEA_REGISTRY_HOST: kankali.darkhelm.lan GITEA_REGISTRY_IP: 10.18.75.2 + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} run: | set -euo pipefail @@ -60,17 +62,78 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi + if [ -n "${PACKAGE_ACCESS_TOKEN:-}" ] && [ -n "${REGISTRY_USER:-}" ]; then + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY_HOST}:3001" -u "${REGISTRY_USER}" --password-stdin >/dev/null || true + fi + + extract_remote_digest_buildx() { + image_ref="$1" + digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{json .Manifest.Digest}}' 2>/dev/null || true)" + digest="${digest//\"/}" + digest="${digest//$'\n'/}" + digest="${digest//$'\r'/}" + digest="${digest// /}" + if [[ "${digest}" == sha256:* ]]; then + printf '%s\n' "${digest}" + return 0 + fi + return 1 + } + + extract_remote_digest_manifest() { + image_ref="$1" + digest="$(docker manifest inspect "${image_ref}" 2>/dev/null | sed -n 's/.*"digest"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1 || true)" + if [[ "${digest}" == sha256:* ]]; then + printf '%s\n' "${digest}" + return 0 + fi + return 1 + } + + remote_digest() { + image_ref="$1" + if [[ "${image_ref}" == *@sha256:* ]]; then + printf '%s\n' "${image_ref##*@}" + return 0 + fi + extract_remote_digest_buildx "${image_ref}" || extract_remote_digest_manifest "${image_ref}" + } + + local_digest_matches() { + image_ref="$1" + digest="$2" + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${image_ref}" 2>/dev/null | grep -q "@${digest}$" + } + + resolve_candidate() { + image_ref="$1" + + if docker image inspect "${image_ref}" >/dev/null 2>&1; then + if digest="$(remote_digest "${image_ref}")"; then + if local_digest_matches "${image_ref}" "${digest}"; then + echo "Using cached current image: ${image_ref}" >&2 + printf '%s\n' "${image_ref}" + return 0 + fi + echo "Cached image is stale for ${image_ref}; refreshing" >&2 + else + echo "Remote digest unavailable for ${image_ref}; attempting refresh pull" >&2 + fi + fi + + if retry_cmd 3 15 docker pull "${image_ref}" >/dev/null; then + printf '%s\n' "${image_ref}" + return 0 + fi + + return 1 + } + pick_renovate_image() { for candidate in "${RENOVATE_IMAGE_PRIMARY}" "${RENOVATE_IMAGE_FALLBACK}"; do echo "Trying Renovate image candidate: ${candidate}" >&2 - if docker image inspect "${candidate}" >/dev/null 2>&1; then - echo "Using cached Renovate image: ${candidate}" >&2 - printf '%s\n' "${candidate}" - return 0 - fi - - if retry_cmd 3 15 docker pull "${candidate}" >/dev/null; then + if resolve_candidate "${candidate}" >/dev/null; then printf '%s\n' "${candidate}" return 0 fi @@ -94,11 +157,6 @@ jobs: echo "RENOVATE_IMAGE=${RENOVATE_IMAGE}" >> "$GITHUB_ENV" - - name: Configure Renovate for Gitea - run: | - echo "=== Configuring Renovate for Gitea ===" - echo "✓ Renovate runtime configuration will be passed through environment and CLI flags" - - name: Run Renovate env: # Prefer dedicated Renovate token, then fall back to existing CI tokens. @@ -490,4 +548,4 @@ jobs: echo "Check the Dependency Dashboard issue in your repository for detailed results:" echo "https://dogar.darkhelm.org/DarkHelm.org/plex-playlist/issues" echo "" - echo "Next scheduled run: Next Monday at 8 AM UTC" + echo "Next scheduled run: Next weekday at 8 AM UTC" diff --git a/Dockerfile.cicd-base b/Dockerfile.cicd-base index d3116e8..5d5bce4 100644 --- a/Dockerfile.cicd-base +++ b/Dockerfile.cicd-base @@ -19,43 +19,47 @@ ENV TZ=America/New_York # Configure timezone RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -# Install apt-fast with proper GPG handling -RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - for i in 1 2 3; do \ - echo "Attempt $i: Updating package lists..." && \ - apt-get update && break || \ - (echo "Update attempt $i failed, retrying..." && sleep 10); \ - done && \ - apt-get install -y \ - software-properties-common \ - gnupg \ - ca-certificates \ - curl \ - wget \ - && for i in 1 2 3; do \ - echo "Attempt $i: Adding apt-fast PPA..." && \ - add-apt-repository -y ppa:apt-fast/stable && \ - apt-get update && \ - apt-get install -y apt-fast && \ - break || \ - (echo "apt-fast installation attempt $i failed, retrying..." && sleep 10); \ - done \ - && rm -rf /var/lib/apt/lists/* - -# Configure apt-fast to use apt (not apt-get) with optimized settings -RUN echo 'apt-fast apt-fast/maxdownloads string 10' | debconf-set-selections && \ - echo 'apt-fast apt-fast/dlflag boolean true' | debconf-set-selections && \ - echo 'apt-fast apt-fast/aptmanager string apt' | debconf-set-selections - # Configure apt timeouts and retries RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \ echo 'Acquire::http::Timeout "60";' >> /etc/apt/apt.conf.d/80retries && \ echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80retries && \ echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80retries -# Install system dependencies using apt-fast -RUN apt-fast update && apt-fast install -y \ +# Bootstrap certificates over HTTP, then enforce HTTPS for all remaining package operations. +RUN set -eux; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/*; \ + find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 \ + | xargs -0 sed -i 's|https://ports.ubuntu.com/ubuntu-ports|http://ports.ubuntu.com/ubuntu-ports|g; s|https://archive.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g; s|https://security.ubuntu.com/ubuntu|http://security.ubuntu.com/ubuntu|g'; \ + for i in 1 2 3; do \ + echo "Attempt $i: Bootstrapping CA certificates..."; \ + if apt-get update && apt-get install -y --no-install-recommends ca-certificates; then \ + break; \ + fi; \ + if [ "$i" -lt 3 ]; then \ + echo "Bootstrap attempt $i failed, retrying..."; \ + sleep 10; \ + else \ + exit 1; \ + fi; \ + done; \ + update-ca-certificates; \ + find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 \ + | xargs -0 sed -i 's|http://ports.ubuntu.com/ubuntu-ports|https://ports.ubuntu.com/ubuntu-ports|g; s|http://archive.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|https://security.ubuntu.com/ubuntu|g'; \ + rm -rf /var/lib/apt/lists/*; \ + for i in 1 2 3; do \ + echo "Attempt $i: Updating package lists over HTTPS..."; \ + if apt-get update; then \ + break; \ + fi; \ + if [ "$i" -lt 3 ]; then \ + echo "HTTPS update attempt $i failed, retrying..."; \ + sleep 10; \ + else \ + exit 1; \ + fi; \ + done; \ + apt-get install -y --no-install-recommends \ git \ curl \ ca-certificates \ @@ -75,12 +79,15 @@ RUN apt-fast update && apt-fast install -y \ libxkbcommon0 \ libasound2 \ tzdata \ + gnupg \ + wget \ && rm -rf /var/lib/apt/lists/* # Install Python 3.14 with retry and fallback mechanisms RUN for i in 1 2 3; do \ echo "Attempt $i: Adding deadsnakes PPA..." && \ add-apt-repository -y ppa:deadsnakes/ppa && \ + find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 | xargs -0 sed -i 's|http://|https://|g' && \ apt-get update && \ break || \ (echo "Attempt $i failed, retrying in 10s..." && sleep 10); \ @@ -88,7 +95,7 @@ RUN for i in 1 2 3; do \ RUN for i in 1 2 3; do \ echo "Attempt $i: Installing Python 3.14..." && \ - timeout 300 apt-fast install -y \ + timeout 300 apt-get install -y --no-install-recommends \ python3.14 \ python3.14-venv \ python3.14-dev && \ @@ -102,8 +109,9 @@ RUN for i in 1 2 3; do \ echo "Attempt $i: Installing Node.js 24..." && \ curl -fsSL --connect-timeout 30 --max-time 300 \ https://deb.nodesource.com/setup_24.x | bash - && \ - apt-fast update && \ - timeout 300 apt-fast install -y nodejs && \ + find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 | xargs -0 sed -i 's|http://|https://|g' && \ + apt-get update && \ + timeout 300 apt-get install -y --no-install-recommends nodejs && \ break || \ (echo "Attempt $i failed, retrying in 15s..." && sleep 15); \ done && \ diff --git a/Dockerfile.frontend b/Dockerfile.frontend index a58a56c..1730797 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -19,8 +19,7 @@ RUN set -eux; \ done; \ return 1; \ }; \ - corepack enable; \ - retry 5 corepack prepare yarn@4.10.3 --activate; \ + retry 5 npm install -g --force @yarnpkg/cli-dist@4.10.3; \ retry 5 yarn install --immutable FROM deps AS build diff --git a/backend/src/backend/main.py b/backend/src/backend/main.py index 2b2ebda..5e09c4f 100644 --- a/backend/src/backend/main.py +++ b/backend/src/backend/main.py @@ -21,10 +21,10 @@ from backend.database import ( ) REQUIRED_PACKAGE_PINS: dict[str, str] = { - "fastapi": "0.120.2", + "fastapi": "0.139.0", "psycopg": "3.2.12", "sqlalchemy": "2.0.44", - "uvicorn": "0.38.0", + "uvicorn": "0.51.0", } diff --git a/backend/tests/integration/test_api.py b/backend/tests/integration/test_api.py index 6c242d4..9ae0e93 100644 --- a/backend/tests/integration/test_api.py +++ b/backend/tests/integration/test_api.py @@ -95,8 +95,8 @@ class TestAPIIntegration: assert "current_python" in payload assert payload["python_policy_valid"] is True assert "required_packages" in payload - assert payload["required_packages"]["fastapi"] == "0.120.2" - assert payload["required_packages"]["uvicorn"] == "0.38.0" + assert payload["required_packages"]["fastapi"] == "0.139.0" + assert payload["required_packages"]["uvicorn"] == "0.51.0" assert payload["package_errors"] == {} monkeypatch.delenv("BACKEND_REQUIRED_PYTHON", raising=False) diff --git a/backend/tests/test_basic.py b/backend/tests/test_basic.py index 06499cf..c0e1e31 100644 --- a/backend/tests/test_basic.py +++ b/backend/tests/test_basic.py @@ -1,12 +1,22 @@ """Basic tests for the backend application.""" -from typing import cast +from importlib import metadata +from typing import Any, cast from unittest.mock import AsyncMock +import pytest from fastapi.testclient import TestClient +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession -from backend.main import app, get_api_session, read_root +from backend.main import ( + app, + compatibility_check, + compatibility_status, + get_api_session, + read_root, + validate_runtime_policy, +) def test_app_creation(): @@ -56,3 +66,84 @@ def test_typeguard_validation(): # This would fail with typeguard active, but we'll just test the happy path # to ensure coverage of our code without breaking the test + + +def test_health_check_db_unavailable(): + """Health endpoint should return unavailable when DB probe fails.""" + + unhealthy_session = cast("AsyncSession", AsyncMock(spec=AsyncSession)) + unhealthy_session.execute = AsyncMock( + side_effect=SQLAlchemyError("database unavailable") + ) + + async def override_get_session(): + """Provide an unhealthy session dependency override for tests.""" + yield unhealthy_session + + app.dependency_overrides[get_api_session] = override_get_session + try: + with TestClient(app) as client: + response = client.get("/health") + + assert response.status_code == 503 + assert response.json() == { + "status": "unhealthy", + "database": "disconnected", + } + finally: + app.dependency_overrides.clear() + + +def test_compatibility_status_success(monkeypatch: pytest.MonkeyPatch): + """Compatibility status should be healthy when all checks match policy.""" + + monkeypatch.setenv("BACKEND_REQUIRED_PYTHON", "3.14") + + def installed_version(package_name: str) -> str: + versions = { + "fastapi": "0.139.0", + "psycopg": "3.2.12", + "sqlalchemy": "2.0.44", + "uvicorn": "0.51.0", + } + return versions[package_name] + + monkeypatch.setattr("backend.main._installed_version", installed_version) + status = cast("dict[str, Any]", compatibility_status()) + + assert status["ok"] is True + assert status["package_errors"] == {} + + +def test_compatibility_status_missing_package(monkeypatch: pytest.MonkeyPatch): + """Compatibility status should record metadata lookup failures.""" + + def missing_version(_: str) -> str: + raise metadata.PackageNotFoundError("missing") + + monkeypatch.setattr("backend.main._installed_version", missing_version) + status = cast("dict[str, Any]", compatibility_status()) + package_checks = cast("dict[str, bool]", status["package_checks"]) + package_errors = cast("dict[str, str]", status["package_errors"]) + + assert status["ok"] is False + assert package_checks["fastapi"] is False + assert "fastapi" in package_errors + + +def test_validate_runtime_policy_raises(monkeypatch: pytest.MonkeyPatch): + """Runtime policy validation should fail when compatibility is not ok.""" + + def invalid_status() -> dict[str, object]: + return {"ok": False} + + monkeypatch.setattr("backend.main.compatibility_status", invalid_status) + + with pytest.raises(RuntimeError): + validate_runtime_policy() + + +def test_compatibility_check_returns_status() -> None: + """Compatibility endpoint helper should return policy payload.""" + payload = compatibility_check() + assert "ok" in payload diff --git a/scripts/resolve-mirrored-image.sh b/scripts/resolve-mirrored-image.sh new file mode 100755 index 0000000..18ce851 --- /dev/null +++ b/scripts/resolve-mirrored-image.sh @@ -0,0 +1,313 @@ +#!/bin/bash + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: resolve-mirrored-image.sh --image [options] + +Required: + --image IMAGE Desired image reference to use locally. + +Optional: + --mirror-image IMAGE Mirror image reference to check/pull/push. + --upstream-image IMAGE Upstream source image to import into mirror when mirror is missing. + --registry-user USER Registry username for mirror auth. + --registry-password-env VAR Environment variable name containing mirror registry password/token. + --disable-remote-digest-check Skip remote digest freshness check for local images. + --local-only Only verify local presence; do not pull. + --quiet Suppress informational output except final image ref. + +Behavior: + 1. If IMAGE exists locally and matches remote digest, use it. + 2. Otherwise try to pull IMAGE. + 3. If IMAGE pull fails and MIRROR_IMAGE is set, resolve via MIRROR_IMAGE. + 4. If MIRROR_IMAGE is missing and UPSTREAM_IMAGE is set, pull upstream, push mirror, then use mirror. + 5. Prints the resolved local image reference on stdout. +EOF +} + +log() { + if [[ "${QUIET}" != "true" ]]; then + echo "$*" >&2 + fi +} + +require_arg() { + local value="$1" + local name="$2" + if [[ -z "${value}" ]]; then + echo "Missing required argument: ${name}" >&2 + usage >&2 + exit 1 + fi +} + +login_if_needed() { + local image_ref="$1" + local registry password_var password + + registry="${image_ref%%/*}" + if [[ -z "${registry}" || "${registry}" == "${image_ref}" ]]; then + return 0 + fi + + if [[ -z "${REGISTRY_USER}" || -z "${REGISTRY_PASSWORD_ENV}" ]]; then + return 0 + fi + + password_var="${REGISTRY_PASSWORD_ENV}" + password="${!password_var:-}" + if [[ -z "${password}" ]]; then + log "Skipping docker login for ${registry}: env ${password_var} is empty" + return 0 + fi + + if [[ -n "${LOGGED_IN_REGISTRIES[${registry}]:-}" ]]; then + return 0 + fi + + log "Logging into ${registry}" + printf '%s' "${password}" | docker login "http://${registry}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + LOGGED_IN_REGISTRIES["${registry}"]=1 +} + +image_present_locally() { + local image_ref="$1" + docker image inspect "${image_ref}" >/dev/null 2>&1 +} + +is_digest_pinned_ref() { + local image_ref="$1" + [[ "${image_ref}" == *@sha256:* ]] +} + +extract_remote_digest_buildx() { + local image_ref="$1" + local digest + + if ! digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{json .Manifest.Digest}}' 2>/dev/null)"; then + return 1 + fi + + digest="${digest//\"/}" + digest="${digest//$'\n'/}" + digest="${digest//$'\r'/}" + digest="${digest// /}" + + if [[ "${digest}" == sha256:* ]]; then + printf '%s\n' "${digest}" + return 0 + fi + + return 1 +} + +extract_remote_digest_manifest() { + local image_ref="$1" + local digest + + if ! digest="$(docker manifest inspect "${image_ref}" 2>/dev/null | sed -n 's/.*"digest"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)"; then + return 1 + fi + + if [[ "${digest}" == sha256:* ]]; then + printf '%s\n' "${digest}" + return 0 + fi + + return 1 +} + +get_remote_digest() { + local image_ref="$1" + + if is_digest_pinned_ref "${image_ref}"; then + printf '%s\n' "${image_ref##*@}" + return 0 + fi + + if extract_remote_digest_buildx "${image_ref}"; then + return 0 + fi + + if extract_remote_digest_manifest "${image_ref}"; then + return 0 + fi + + return 1 +} + +local_image_matches_digest() { + local image_ref="$1" + local digest="$2" + + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${image_ref}" 2>/dev/null | grep -q "@${digest}$" +} + +pull_image() { + local image_ref="$1" + login_if_needed "${image_ref}" + docker pull "${image_ref}" >/dev/null +} + +resolve_reference() { + local image_ref="$1" + local remote_digest + + if image_present_locally "${image_ref}"; then + if [[ "${DISABLE_REMOTE_DIGEST_CHECK}" == "true" ]] || [[ "${LOCAL_ONLY}" == "true" ]] || is_digest_pinned_ref "${image_ref}"; then + log "Using locally cached image ${image_ref}" + return 0 + fi + + login_if_needed "${image_ref}" + if remote_digest="$(get_remote_digest "${image_ref}")"; then + if local_image_matches_digest "${image_ref}" "${remote_digest}"; then + log "Using local image ${image_ref}; remote digest matches (${remote_digest})" + return 0 + fi + log "Local image ${image_ref} is stale; expected remote digest ${remote_digest}" + else + log "Unable to determine remote digest for ${image_ref}; attempting pull refresh" + fi + fi + + if [[ "${LOCAL_ONLY}" == "true" ]]; then + return 1 + fi + + if pull_image "${image_ref}"; then + log "Pulled image ${image_ref}" + return 0 + fi + + return 1 +} + +mirror_available() { + local image_ref="$1" + if pull_image "${image_ref}"; then + return 0 + fi + return 1 +} + +promote_upstream_to_mirror() { + require_arg "${UPSTREAM_IMAGE}" "--upstream-image" + require_arg "${MIRROR_IMAGE}" "--mirror-image" + + log "Pulling upstream image ${UPSTREAM_IMAGE}" + docker pull "${UPSTREAM_IMAGE}" >/dev/null + + login_if_needed "${MIRROR_IMAGE}" + log "Tagging ${UPSTREAM_IMAGE} as ${MIRROR_IMAGE}" + docker tag "${UPSTREAM_IMAGE}" "${MIRROR_IMAGE}" + log "Pushing ${MIRROR_IMAGE}" + docker push "${MIRROR_IMAGE}" >/dev/null +} + +IMAGE="" +MIRROR_IMAGE="" +UPSTREAM_IMAGE="" +REGISTRY_USER="" +REGISTRY_PASSWORD_ENV="" +DISABLE_REMOTE_DIGEST_CHECK="false" +LOCAL_ONLY="false" +QUIET="false" +declare -A LOGGED_IN_REGISTRIES=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --image) + IMAGE="${2:-}" + shift 2 + ;; + --mirror-image) + MIRROR_IMAGE="${2:-}" + shift 2 + ;; + --upstream-image) + UPSTREAM_IMAGE="${2:-}" + shift 2 + ;; + --registry-user) + REGISTRY_USER="${2:-}" + shift 2 + ;; + --registry-password-env) + REGISTRY_PASSWORD_ENV="${2:-}" + shift 2 + ;; + --disable-remote-digest-check) + DISABLE_REMOTE_DIGEST_CHECK="true" + shift + ;; + --local-only) + LOCAL_ONLY="true" + shift + ;; + --quiet) + QUIET="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +require_arg "${IMAGE}" "--image" + +if resolve_reference "${IMAGE}"; then + printf '%s\n' "${IMAGE}" + exit 0 +fi + +if [[ "${LOCAL_ONLY}" == "true" ]]; then + echo "Image unavailable via local-only mode: ${IMAGE}" >&2 + exit 1 +fi + +if [[ -n "${MIRROR_IMAGE}" ]]; then + log "Primary image unavailable; checking mirror ${MIRROR_IMAGE}" + if resolve_reference "${MIRROR_IMAGE}"; then + printf '%s\n' "${MIRROR_IMAGE}" + exit 0 + fi + + if [[ -n "${UPSTREAM_IMAGE}" ]]; then + log "Mirror image ${MIRROR_IMAGE} missing; promoting from upstream ${UPSTREAM_IMAGE}" + promote_upstream_to_mirror + if ! pull_image "${MIRROR_IMAGE}"; then + echo "Failed to pull promoted mirror image: ${MIRROR_IMAGE}" >&2 + exit 1 + fi + printf '%s\n' "${MIRROR_IMAGE}" + exit 0 + fi +fi + +if [[ -n "${UPSTREAM_IMAGE}" ]]; then + log "Falling back to upstream image ${UPSTREAM_IMAGE}" + if resolve_reference "${UPSTREAM_IMAGE}"; then + printf '%s\n' "${UPSTREAM_IMAGE}" + exit 0 + fi +fi + +echo "Failed to resolve image via primary/mirror/upstream path" >&2 +echo "primary=${IMAGE}" >&2 +if [[ -n "${MIRROR_IMAGE}" ]]; then + echo "mirror=${MIRROR_IMAGE}" >&2 +fi +if [[ -n "${UPSTREAM_IMAGE}" ]]; then + echo "upstream=${UPSTREAM_IMAGE}" >&2 +fi +exit 1