name: CICD

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main, develop]
  workflow_dispatch:
    inputs:
      head_sha:
        description: Commit SHA to process
        required: false
      force_rebuild_base:
        description: Force CICD base rebuild
        required: false
        default: "false"

env:
  GITEA_SSH_HOST: kankali.darkhelm.lan
  GITEA_SSH_PORT: "2222"
  GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
  GITEA_REGISTRY: kankali.darkhelm.lan:3001
  GITEA_REGISTRY_IP: 10.18.75.2
  GITEA_REGISTRY_HOST: kankali.darkhelm.lan
  GITEA_REGISTRY_AUTH_HOST: kankali.darkhelm.lan
  GITEA_REGISTRY_AUTH_FALLBACK_HOST: dogar.darkhelm.org
  CI_PRUNE_AT_JOB_START: "true"
  CI_PRUNE_THRESHOLD_PERCENT: "90"

defaults:
  run:
    shell: bash

concurrency:
  group: cicd-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_cicd:
    name: Build and Push CICD Images
    runs-on: ubuntu-act-8gb
    timeout-minutes: 35
    outputs:
      base_hash: ${{ steps.base-state.outputs.base_hash }}
      head_sha: ${{ steps.meta.outputs.head_sha }}
    steps:
      - &identify_runner_step
        name: Identify runner
        run: |
          echo "=== Runner Identity ==="
          echo "runner_name=${RUNNER_NAME:-}"
          echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
          echo "runner_hostname_env=${HOSTNAME:-unknown}"
          echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
          echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
          echo "runner_os=${RUNNER_OS:-unknown}"
          echo "runner_arch=${RUNNER_ARCH:-unknown}"
          echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"

      - &pre_job_prune_step
        name: Pre-job Docker prune
        run: |
          if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then
            echo "Skipping pre-job prune"
            exit 0
          fi

          used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
          threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
          echo "disk_used_before=${used_before}%"
          echo "prune_threshold=${threshold}%"

          if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
            docker builder prune -af || true
            docker system prune -af --volumes || true
            used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
            echo "disk_used_after=${used_after}%"
          else
            echo "Skipping prune; usage below threshold"
          fi

      - name: Resolve head SHA
        id: meta
        env:
          HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
          HEAD_SHA_FALLBACK: ${{ github.sha }}
        run: |
          RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
          echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"

      - name: Minimal checkout for base inputs
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail
          umask 077
          trap 'rm -f ~/.ssh/id_rsa' EXIT

          retry_cmd() {
            attempts="${1:-5}"
            backoff="${2:-2}"
            shift 2

            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if "$@"; then
                return 0
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Command failed (attempt ${attempt}/${attempts}): $*"
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            echo "Command failed after ${attempts} attempts: $*"
            return 1
          }

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          mkdir -p ~/.ssh
          echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"

          retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
            git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .

          if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
            git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
            git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
          else
            git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
          fi

          chmod +x scripts/compute-cicd-base-hash.sh
          chmod +x scripts/resolve-mirrored-image.sh

      - name: Compute base hash and inspect registry
        id: base-state
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          FORCE_REBUILD: ${{ github.event.inputs.force_rebuild_base }}
        run: |
          set -euo pipefail

          ensure_registry_auth_realm_host() {
            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
            fi

            if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
            fi

            headers_file="$(mktemp)"
            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
              if [ -n "${realm_url}" ]; then
                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
                  if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
                    echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
                  fi
                  if ! grep -q "${realm_host}" /etc/hosts; then
                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
                  fi
                fi
              fi
            fi
            rm -f "${headers_file}"
          }

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              ensure_registry_auth_realm_host
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
          BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
          BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest"

          echo "base_hash=${BASE_HASH}" >> "$GITHUB_OUTPUT"
          echo "base_ref_hash=${BASE_REF_HASH}" >> "$GITHUB_OUTPUT"
          echo "base_ref_latest=${BASE_REF_LATEST}" >> "$GITHUB_OUTPUT"

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          ensure_registry_auth_realm_host

          if ! docker_login_with_retry 5 3 >/dev/null; then
            echo "❌ Failed docker login after retries"
            exit 1
          fi

          if [ "${FORCE_REBUILD:-false}" = "true" ]; then
            echo "needs_build=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          if timeout 900 docker pull "${BASE_REF_HASH}" >/dev/null 2>&1; then
            echo "needs_build=false" >> "$GITHUB_OUTPUT"
          else
            echo "needs_build=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Build and push base image
        if: steps.base-state.outputs.needs_build == 'true'
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          BASE_HASH: ${{ steps.base-state.outputs.base_hash }}
          BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }}
          BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }}
        run: |
          set -euo pipefail

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          ensure_registry_auth_realm_host() {
            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
            fi

            if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
            fi

            headers_file="$(mktemp)"
            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
              if [ -n "${realm_url}" ]; then
                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
                  if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
                    echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
                  fi
                  if ! grep -q "${realm_host}" /etc/hosts; then
                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
                  fi
                fi
              fi
            fi
            rm -f "${headers_file}"
          }

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              ensure_registry_auth_realm_host
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          docker_login_with_retry 5 3

          PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
          PLAYWRIGHT_BROWSERS_UPSTREAM_TAG="mcr.microsoft.com/playwright:v1.56.1-jammy"
          PLAYWRIGHT_BROWSERS_LOCAL_TAG="$(./scripts/resolve-mirrored-image.sh \
            --image "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" \
            --mirror-image "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" \
            --upstream-image "${PLAYWRIGHT_BROWSERS_UPSTREAM_TAG}" \
            --registry-user "${REGISTRY_USER}" \
            --registry-password-env PACKAGE_ACCESS_TOKEN)"

          PLAYWRIGHT_BROWSERS_IMAGE="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${PLAYWRIGHT_BROWSERS_LOCAL_TAG}" | head -n 1; } || true)"
          if [ -z "${PLAYWRIGHT_BROWSERS_IMAGE}" ]; then
            echo "❌ Unable to resolve Playwright browsers digest reference"
            exit 1
          fi

          docker build --progress=plain -f Dockerfile.cicd-base \
            --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \
            --build-arg BASE_IMAGE_HASH="${BASE_HASH}" \
            --build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \
            -t cicd-base:latest .

          docker tag cicd-base:latest "${BASE_REF_HASH}"
          docker tag cicd-base:latest "${BASE_REF_LATEST}"
          retry_registry_op push "${BASE_REF_HASH}" 5 4
          retry_registry_op push "${BASE_REF_LATEST}" 5 4

      - name: Minimal checkout for CICD build inputs
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail
          umask 077
          trap 'rm -f ~/.ssh/id_rsa' EXIT

          retry_cmd() {
            attempts="${1:-5}"
            backoff="${2:-2}"
            shift 2

            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if "$@"; then
                return 0
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Command failed (attempt ${attempt}/${attempts}): $*"
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            echo "Command failed after ${attempts} attempts: $*"
            return 1
          }

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          mkdir -p ~/.ssh
          echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"

          if [ ! -d .git ]; then
            retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
              git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
          fi

          if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
            git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
            git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
          else
            git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
          fi
          chmod +x scripts/compute-cicd-base-hash.sh
          chmod +x scripts/resolve-mirrored-image.sh

      - name: Build and push complete CICD image
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
          BASE_HASH_FROM_BUILD: ${{ steps.base-state.outputs.base_hash }}
        run: |
          set -euo pipefail
          umask 077
          trap 'rm -f /tmp/ssh_key' EXIT

          trim_spaces() {
            printf '%s' "$1" | tr -d '[:space:]'
          }

          is_hex() {
            value="$1"
            if [ -z "${value}" ]; then
              return 1
            fi
            case "${value}" in
              (*[!0-9a-fA-F]*) return 1 ;;
              (*) return 0 ;;
            esac
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          ensure_registry_auth_realm_host() {
            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
            fi

            if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
            fi

            headers_file="$(mktemp)"
            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
              if [ -n "${realm_url}" ]; then
                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
                  if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
                    echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
                  fi
                  if ! grep -q "${realm_host}" /etc/hosts; then
                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
                  fi
                fi
              fi
            fi
            rm -f "${headers_file}"
          }

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              ensure_registry_auth_realm_host
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          echo "=== Pre-build disk telemetry ==="
          df -h || true
          if command -v docker >/dev/null 2>&1; then
            docker system df || true
            used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
            threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
            echo "disk_used_before=${used_before}%"
            echo "prune_threshold=${threshold}%"
            if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
              docker builder prune -af || true
              docker system prune -af --volumes || true
            else
              echo "Skipping prune; usage below threshold"
            fi
            echo "=== Post-prune disk telemetry ==="
            df -h || true
            docker system df || true
          fi

          ensure_registry_auth_realm_host

          docker_login_with_retry 5 3

          RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")"
          if ! is_hex "${RESOLVED_HEAD_SHA}"; then
            echo "❌ Invalid or empty HEAD_SHA resolved for build_cicd: '${HEAD_SHA:-}'"
            exit 1
          fi

          BASE_HASH="$(trim_spaces "${BASE_HASH_FROM_BUILD}")"
          if ! is_hex "${BASE_HASH}"; then
            BASE_HASH="$(./scripts/compute-cicd-base-hash.sh | tr -d '[:space:]')"
          fi

          if ! is_hex "${BASE_HASH}"; then
            echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_FROM_BUILD}'"
            exit 1
          fi

          echo "resolved_head_sha=${RESOLVED_HEAD_SHA}"
          echo "resolved_base_hash=${BASE_HASH}"

          BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
          CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest"
          CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}"

          retry_registry_op pull "${BASE_IMAGE}" 5 3
          retry_registry_op pull "${CICD_LATEST_REF}" 5 3 || true

          echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
          chmod 600 /tmp/ssh_key

          DOCKER_BUILDKIT=1 docker build -f Dockerfile.cicd \
            --cache-from "${CICD_LATEST_REF}" \
            --secret id=ssh_private_key,src=/tmp/ssh_key \
            --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            --build-arg GITHUB_SHA="${RESOLVED_HEAD_SHA}" \
            --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \
            -t cicd:latest .

          docker tag cicd:latest "${CICD_LATEST_REF}"
          docker tag cicd:latest "${CICD_SHA_REF}"
          retry_registry_op push "${CICD_LATEST_REF}" 5 4
          retry_registry_op push "${CICD_SHA_REF}" 5 4

      - &failure_diagnostics_step
        name: Failure diagnostics
        if: failure()
        run: |
          echo "=== Failure Diagnostics ==="
          date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
          echo "runner_name=${RUNNER_NAME:-unknown}"
          echo "runner_hostname=${HOSTNAME:-unknown}"
          uname -a || true
          cat /etc/os-release 2>/dev/null || true
          df -h || true
          free -h || true
          ps aux --sort=-%mem | head -n 30 || true
          if command -v docker >/dev/null 2>&1; then
            echo "=== Docker Diagnostics ==="
            docker version || true
            docker info || true
            docker ps -a || true
            docker images --digests | head -n 50 || true
          fi
          dmesg | tail -n 120 || true

  build-cicd-postmortem:
    name: Build CICD Image Failure Postmortem
    runs-on: ubuntu-act
    needs: build_cicd
    if: always() && needs.build_cicd.result == 'failure'
    timeout-minutes: 10
    steps:
      - *identify_runner_step

      - name: Build CICD postmortem diagnostics
        run: |
          echo "=== Build CICD Postmortem ==="
          echo "build_cicd_result=${{ needs.build_cicd.result }}"
          uname -a || true
          cat /etc/os-release 2>/dev/null || true
          df -h || true
          free -h || true
          if command -v docker >/dev/null 2>&1; then
            docker version || true
            docker info || true
          fi
          timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true

  source-checks:
    name: Source Checks
    runs-on: ubuntu-act
    timeout-minutes: 75
    needs: build_cicd
    steps:
      - *identify_runner_step

      - &configure_registry_host_step
        name: Configure registry host resolution
        run: |
          ensure_registry_auth_realm_host() {
            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
            fi

            if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
            fi

            headers_file="$(mktemp)"
            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
              if [ -n "${realm_url}" ]; then
                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
                  if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
                    echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
                  fi
                  if ! grep -q "${realm_host}" /etc/hosts; then
                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
                  fi
                fi
              fi
            fi
            rm -f "${headers_file}"
          }

          if [ "${CI_PRUNE_AT_JOB_START:-false}" = "true" ] && command -v docker >/dev/null 2>&1; then
            used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
            threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
            echo "disk_used_before=${used_before}%"
            echo "prune_threshold=${threshold}%"
            if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
              docker builder prune -af || true
              docker system prune -af --volumes || true
              used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
              echo "disk_used_after=${used_after}%"
            else
              echo "Skipping prune; usage below threshold"
            fi
          fi

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          ensure_registry_auth_realm_host

      - &ensure_cicd_image_step
        name: Ensure CICD image is available
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || github.sha }}
        run: |
          IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
          if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
            echo "Using cached CICD image: ${IMAGE}"
          else
            login_ok=false
            for i in 1 2 3 4 5; do
              if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
                login_ok=true
                break
              fi
              if [ "${i}" -lt 5 ]; then
                sleep_seconds=$((3 * i))
                echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
            done

            if [ "${login_ok}" != "true" ]; then
              echo "❌ Failed docker login after retries"
              exit 1
            fi

            pulled=false
            for i in 1 2 3; do
              echo "Pull attempt ${i}/3 for ${IMAGE}"
              if docker pull "${IMAGE}"; then
                pulled=true
                break
              fi
              if [ "${i}" -lt 3 ]; then
                sleep_seconds=$((5 * i))
                echo "Pull failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
            done

            if [ "${pulled}" != "true" ]; then
              echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
              exit 1
            fi
          fi

      - name: Run backend tests with coverage
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
        run: |
          set -o pipefail
          LOG_FILE="$(mktemp)"

          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
            cd /workspace/backend &&
            source .venv/bin/activate &&
            uv run pytest -v --tb=short -m 'not integration' --cov=src --cov-report=term-missing --cov-fail-under=95
          " 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "❌ Backend tests failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of backend test output ---"
            tail -n 200 "${LOG_FILE}" || true
            exit "${TEST_STATUS}"
          fi

      - name: Run pre-commit checks
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
        run: |
          set -o pipefail
          LOG_FILE="$(mktemp)"

          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
            cd /workspace &&
            if [ ! -d .git ]; then
              git init /workspace
            fi &&
            /workspace/backend/.venv/bin/pre-commit run --all-files --show-diff-on-failure --config .pre-commit-config.yaml
          " 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "❌ Pre-commit checks failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of pre-commit output ---"
            tail -n 200 "${LOG_FILE}" || true
            exit "${TEST_STATUS}"
          fi

      - name: Run frontend tests with coverage
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
        run: |
          set -o pipefail
          LOG_FILE="$(mktemp)"

          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
            cd /workspace/frontend &&
            yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85
          " 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "❌ Frontend tests failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of frontend test output ---"
            tail -n 200 "${LOG_FILE}" || true
            exit "${TEST_STATUS}"
          fi

      - name: Run backend doctests
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
        run: |
          set -o pipefail
          LOG_FILE="$(mktemp)"

          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
            cd /workspace/backend &&
            .venv/bin/python -m xdoctest src/ --quiet
          " 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "❌ Backend doctests failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of doctest output ---"
            tail -n 200 "${LOG_FILE}" || true
            exit "${TEST_STATUS}"
          fi

      - *failure_diagnostics_step

  dependency-audits:
    name: Dependency Audits (Informational)
    runs-on: ubuntu-act
    continue-on-error: true
    timeout-minutes: 15
    needs: build_cicd
    steps:
      - *identify_runner_step

      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run frontend dependency audit
        continue-on-error: true
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
        run: |
          set -o pipefail
          LOG_FILE="$(mktemp)"

          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
            cd /workspace &&
            uv --directory backend run poe audit-frontend
          " 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "⚠️ Frontend dependency audit failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of frontend audit output ---"
            tail -n 200 "${LOG_FILE}" || true
            echo "Proceeding: frontend audit is informational-only."
          fi

      - name: Run backend dependency audit
        if: always()
        continue-on-error: true
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
        run: |
          set -o pipefail
          LOG_FILE="$(mktemp)"

          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
            cd /workspace &&
            uv --directory backend run poe audit-backend
          " 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "⚠️ Backend dependency audit failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of backend audit output ---"
            tail -n 200 "${LOG_FILE}" || true
            echo "Proceeding: backend audit is informational-only."
          fi

      - *failure_diagnostics_step

  cicd-tests-complete:
    name: CICD Tests Complete
    runs-on: ubuntu-act
    needs: [source-checks, dependency-audits]
    if: always()
    steps:
      - name: Confirm all source lanes passed
        run: |
          set -euo pipefail
          source_checks_status="${{ needs['source-checks'].result }}"
          dependency_audits_status="${{ needs['dependency-audits'].result }}"

          echo "source-checks=${source_checks_status}"
          echo "dependency-audits=${dependency_audits_status}"

          if [ "${source_checks_status}" != "success" ]; then
            echo "❌ One or more CICD source lanes failed"
            exit 1
          fi

          if [ "${dependency_audits_status}" != "success" ]; then
            echo "⚠️ Dependency audit lane reported non-success (informational only)"
          fi

          echo "✅ Source checks complete"

      - *failure_diagnostics_step

  build-release-images:
    name: Build Release Images
    runs-on: ubuntu-act-8gb
    needs: [build_cicd, source-checks]
    outputs:
      head_sha: ${{ steps.meta.outputs.head_sha }}
      backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }}
      backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }}
      frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }}
      frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }}
      deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
      deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
      deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
      deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
    steps:
      - &resolve_head_sha_from_build_step
        name: Resolve head SHA
        id: meta
        run: |
          echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT"

      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - &hydrate_build_context_from_cicd_step
        name: Populate build context from CICD image
        env:
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail
          SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
          SOURCE_CONTAINER="$(docker create "${SOURCE_IMAGE}")"
          cleanup() {
            docker rm -f "${SOURCE_CONTAINER}" >/dev/null 2>&1 || true
          }
          trap cleanup EXIT

          docker cp "${SOURCE_CONTAINER}:/workspace/." .

      - name: Build and push backend base image
        id: backend_base_ref
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          docker_login_with_retry 5 3
          BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base"
          BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}"
          BACKEND_BASE_CACHE_REF="${BACKEND_BASE_REPO}:cache"

          retry_registry_op pull "${BACKEND_BASE_CACHE_REF}" 5 3 || true
          DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
            --target dependencies \
            --cache-from "${BACKEND_BASE_CACHE_REF}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            -t plex-playlist-backend-base:"${HEAD_SHA}" .
          docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}"
          docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_CACHE_REF}"
          retry_registry_op push "${BACKEND_BASE_TAG_REF}" 5 4
          retry_registry_op push "${BACKEND_BASE_CACHE_REF}" 5 4
          retry_registry_op pull "${BACKEND_BASE_TAG_REF}" 5 3

          BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)"
          if [ -z "${BACKEND_BASE_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve backend base image digest"
            exit 1
          fi

          echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
          echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"

      - name: Build and push frontend base image
        id: frontend_base_ref
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail

          retry_cmd() {
            attempts="${1:-5}"
            backoff="${2:-2}"
            shift 2

            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if "$@"; then
                return 0
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Command failed (attempt ${attempt}/${attempts}): $*"
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            echo "Command failed after ${attempts} attempts: $*"
            return 1
          }

          docker_login_with_retry() {
            retry_cmd "${1:-5}" "${2:-3}" sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin'
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          docker_login_with_retry 5 3
          FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base"
          FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}"
          FRONTEND_BASE_CACHE_REF="${FRONTEND_BASE_REPO}:cache"

          retry_registry_op pull "${FRONTEND_BASE_CACHE_REF}" 5 3 || true
          BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
            --target deps \
            --cache-from "${FRONTEND_BASE_CACHE_REF}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            -t plex-playlist-frontend-base:"${HEAD_SHA}" .
          docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}"
          docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_CACHE_REF}"
          retry_registry_op push "${FRONTEND_BASE_TAG_REF}" 5 4
          retry_registry_op push "${FRONTEND_BASE_CACHE_REF}" 5 4
          retry_registry_op pull "${FRONTEND_BASE_TAG_REF}" 5 3

          FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)"
          if [ -z "${FRONTEND_BASE_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve frontend base image digest"
            exit 1
          fi

          echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
          echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"

      - name: Build and push backend main image
        id: deployable_backend_ref
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-staging"
          DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
          DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache"

          docker_login_with_retry 5 3
          retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true
          DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
            --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            -t plex-playlist-backend:"${HEAD_SHA}" .
          bash ./scripts/verify-deployable-image-purity.sh --image plex-playlist-backend:"${HEAD_SHA}" --profile backend

          docker tag "plex-playlist-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
          docker tag "plex-playlist-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}"

          retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
          retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4
          retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3

          DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
          if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve backend main digest"
            exit 1
          fi

          echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
          echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"

      - name: Build and push frontend main image
        id: deployable_frontend_ref
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail

          ensure_registry_auth_realm_host() {
            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
            fi

            headers_file="$(mktemp)"
            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
              if [ -n "${realm_url}" ]; then
                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
                  if ! grep -q "${realm_host}" /etc/hosts; then
                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
                  fi
                fi
              fi
            fi
            rm -f "${headers_file}"
          }

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              ensure_registry_auth_realm_host
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-staging"
          DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
          DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache"

          docker_login_with_retry 5 3
          retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true
          BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
            --target production \
            --cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            -t plex-playlist-frontend:"${HEAD_SHA}" .
          bash ./scripts/verify-deployable-image-purity.sh --image plex-playlist-frontend:"${HEAD_SHA}" --profile frontend

          docker tag "plex-playlist-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
          docker tag "plex-playlist-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}"

          retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
          retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4
          retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3

          DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
          if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve frontend main digest"
            exit 1
          fi

          echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
          echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"

      - *failure_diagnostics_step

  build-tester-images:
    name: Build Tester Images
    runs-on: ubuntu-act-8gb
    needs: [build_cicd, source-checks]
    outputs:
      integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }}
      integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }}
      e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }}
      e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }}
    steps:
      - *resolve_head_sha_from_build_step
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - *hydrate_build_context_from_cicd_step

      - name: Build and push integration tester image
        id: integration_tester_ref
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail

          ensure_registry_auth_realm_host() {
            if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
              echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
            fi

            headers_file="$(mktemp)"
            if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
              realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
              if [ -n "${realm_url}" ]; then
                realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
                if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
                  if ! grep -q "${realm_host}" /etc/hosts; then
                    echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
                    echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
                  fi
                fi
              fi
            fi
            rm -f "${headers_file}"
          }

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              ensure_registry_auth_realm_host
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          docker_login_with_retry 5 3
          INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration"
          INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}"
          INTEGRATION_TESTER_CACHE_REF="${INTEGRATION_TESTER_REPO}:cache"

          retry_registry_op pull "${INTEGRATION_TESTER_CACHE_REF}" 5 3 || true
          DOCKER_BUILDKIT=1 docker build -f Dockerfile.integration-tester \
            --cache-from "${INTEGRATION_TESTER_CACHE_REF}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            -t plex-playlist-integration:"${HEAD_SHA}" .
          docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}"
          docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_CACHE_REF}"

          retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4
          retry_registry_op push "${INTEGRATION_TESTER_CACHE_REF}" 5 4
          retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3

          INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)"
          if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve integration tester digest"
            exit 1
          fi

          echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
          echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"

      - name: Build and push E2E tester image
        id: e2e_tester_ref
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
        run: |
          set -euo pipefail

          PLAYWRIGHT_BASE_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
          PLAYWRIGHT_BASE_IMAGE_UPSTREAM="mcr.microsoft.com/playwright:v1.56.1-jammy"

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          retry_base_pull() {
            image_ref="$1"
            attempts="${2:-6}"
            backoff="${3:-5}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "base pull attempt ${attempt}/${attempts} for ${image_ref}"

              if getent hosts "${GITEA_REGISTRY_HOST}" >/dev/null 2>&1; then
                echo "${GITEA_REGISTRY_HOST} DNS resolution: ok"
              else
                echo "${GITEA_REGISTRY_HOST} DNS resolution: failed"
              fi

              timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" >/dev/null || true

              if docker pull "${image_ref}"; then
                return 0
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying base image pull in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          docker_login_with_retry 5 3
          E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e"
          E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}"
          E2E_TESTER_CACHE_REF="${E2E_TESTER_REPO}:cache"

          PLAYWRIGHT_BASE_IMAGE="$(./scripts/resolve-mirrored-image.sh \
            --image "${PLAYWRIGHT_BASE_IMAGE_MIRROR}" \
            --mirror-image "${PLAYWRIGHT_BASE_IMAGE_MIRROR}" \
            --upstream-image "${PLAYWRIGHT_BASE_IMAGE_UPSTREAM}" \
            --registry-user "${REGISTRY_USER}" \
            --registry-password-env PACKAGE_ACCESS_TOKEN)"

          retry_base_pull "${PLAYWRIGHT_BASE_IMAGE}" 6 5
          retry_registry_op pull "${E2E_TESTER_CACHE_REF}" 5 3 || true
          DOCKER_BUILDKIT=1 docker build --pull=false -f Dockerfile.e2e-tester \
            --cache-from "${E2E_TESTER_CACHE_REF}" \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            --build-arg PLAYWRIGHT_BASE_IMAGE="${PLAYWRIGHT_BASE_IMAGE}" \
            -t plex-playlist-e2e:"${HEAD_SHA}" .
          docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}"
          docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_CACHE_REF}"

          retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4
          retry_registry_op push "${E2E_TESTER_CACHE_REF}" 5 4
          retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3

          E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)"
          if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve E2E tester digest"
            exit 1
          fi

          echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
          echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"

      - *failure_diagnostics_step

  production-images-complete:
    name: Production Images Complete
    runs-on: ubuntu-act
    needs: [build-release-images, build-tester-images]
    if: always()
    timeout-minutes: 10
    steps:
      - name: Confirm required production/tester images are complete
        run: |
          set -euo pipefail
          release_images_status="${{ needs['build-release-images'].result }}"
          tester_images_status="${{ needs['build-tester-images'].result }}"

          echo "build-release-images=${release_images_status}"
          echo "build-tester-images=${tester_images_status}"

          if [ "${release_images_status}" != "success" ] || [ "${tester_images_status}" != "success" ]; then
            echo "❌ Production Images Complete gate failed"
            exit 1
          fi

          echo "✅ Production, integration tester, and E2E tester images are complete"

      - *failure_diagnostics_step

  runtime-images-postmortem:
    name: Production Image Failures Postmortem
    # Run on the broader runner pool so we can still capture diagnostics when
    # one of the active production image jobs fails during platform setup.
    runs-on: ubuntu-act
    needs: [build-release-images, build-tester-images]
    if: always() && (needs['build-release-images'].result == 'failure' || needs['build-tester-images'].result == 'failure')
    timeout-minutes: 10
    steps:
      - *identify_runner_step

      - name: Collect production image postmortem context
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set +e

          echo "=== Postmortem Context ==="
          echo "workflow=${GITHUB_WORKFLOW:-unknown}"
          echo "run_id=${GITHUB_RUN_ID:-unknown}"
          echo "run_number=${GITHUB_RUN_NUMBER:-unknown}"
          echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
          echo "repository=${GITHUB_REPOSITORY:-unknown}"
          echo "server_url=${GITHUB_SERVER_URL:-unknown}"
          echo "build_release_images_result=${{ needs['build-release-images'].result }}"
          echo "build_tester_images_result=${{ needs['build-tester-images'].result }}"

          echo "=== Local Runner Telemetry (postmortem job host) ==="
          uname -a || true
          cat /etc/os-release 2>/dev/null || true
          df -h || true
          free -h || true
          if command -v docker >/dev/null 2>&1; then
            docker version || true
            docker info || true
          fi

          echo "=== Registry Reachability Probe ==="
          timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true

          if [ -z "${GITHUB_TOKEN:-}" ]; then
            echo "No GITHUB_TOKEN available for API diagnostics"
            exit 0
          fi

          API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
          JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200"

          echo "=== Actions API Job Summary ==="
          echo "jobs_url=${JOBS_URL}"
          if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then
            echo "Could not fetch job metadata from Actions API"
            exit 0
          fi

            if command -v python3 >/dev/null 2>&1; then
            python3 -c "$(printf '%s\n' \
              'import json' \
              'from pathlib import Path' \
              '' \
              'path = Path("/tmp/actions-jobs.json")' \
              'payload = json.loads(path.read_text())' \
              'jobs = payload.get("jobs") or payload.get("data") or []' \
              '' \
              'print("job_count=", len(jobs))' \
              '' \
              'def _g(job, *keys):' \
              '    for key in keys:' \
              '        if key in job and job[key] not in (None, ""):' \
              '            return job[key]' \
              '    return "unknown"' \
              '' \
              'for job in jobs:' \
              '    name = _g(job, "name", "job_name")' \
              '    status = _g(job, "status")' \
              '    conclusion = _g(job, "conclusion", "result")' \
              '    runner = _g(job, "runner_name", "runner")' \
              '    started = _g(job, "started_at", "start_time")' \
              '    completed = _g(job, "completed_at", "end_time")' \
              '    print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \
              '' \
              'targets = {' \
              '    "Build Release Images",' \
              '    "Build Tester Images",' \
              '}' \
              '' \
              'print("=== targeted production image jobs ===")' \
              'for job in jobs:' \
              '    name = str(_g(job, "name", "job_name"))' \
              '    if name in targets:' \
              '        for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \
              '            if key in job:' \
              '                print(f"{key}={job[key]}")' \
              '        print("---")')"
          else
            echo "python3 unavailable; emitting raw jobs payload head"
            sed -n '1,120p' /tmp/actions-jobs.json || true
          fi

          exit 0

  source-lanes-postmortem:
    name: Source Lanes Failure Postmortem
    # Capture follow-up diagnostics when any source lane dies during platform
    # setup before step logging is available.
    runs-on: ubuntu-act
    needs: [source-checks]
    if: always() && needs['source-checks'].result == 'failure'
    timeout-minutes: 10
    steps:
      - *identify_runner_step

      - name: Collect source lane postmortem context
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set +e

          echo "=== Postmortem Context ==="
          echo "workflow=${GITHUB_WORKFLOW:-unknown}"
          echo "run_id=${GITHUB_RUN_ID:-unknown}"
          echo "run_number=${GITHUB_RUN_NUMBER:-unknown}"
          echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
          echo "repository=${GITHUB_REPOSITORY:-unknown}"
          echo "server_url=${GITHUB_SERVER_URL:-unknown}"
          echo "source_checks_result=${{ needs['source-checks'].result }}"

          echo "=== Local Runner Telemetry (postmortem job host) ==="
          uname -a || true
          cat /etc/os-release 2>/dev/null || true
          df -h || true
          free -h || true
          if command -v docker >/dev/null 2>&1; then
            docker version || true
            docker info || true
          fi

          echo "=== Registry Reachability Probe ==="
          timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true

          if [ -z "${GITHUB_TOKEN:-}" ]; then
            echo "No GITHUB_TOKEN available for API diagnostics"
            exit 0
          fi

          API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
          JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200"

          echo "=== Actions API Job Summary ==="
          echo "jobs_url=${JOBS_URL}"
          if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then
            echo "Could not fetch job metadata from Actions API"
            exit 0
          fi

          if command -v python3 >/dev/null 2>&1; then
            python3 -c "$(printf '%s\n' \
              'import json' \
              'from pathlib import Path' \
              '' \
              'path = Path("/tmp/actions-jobs.json")' \
              'payload = json.loads(path.read_text())' \
              'jobs = payload.get("jobs") or payload.get("data") or []' \
              '' \
              'print("job_count=", len(jobs))' \
              '' \
              'def _g(job, *keys):' \
              '    for key in keys:' \
              '        if key in job and job[key] not in (None, ""):' \
              '            return job[key]' \
              '    return "unknown"' \
              '' \
              'targets = {' \
              '    "Source Checks",' \
              '}' \
              '' \
              'for job in jobs:' \
              '    name = str(_g(job, "name", "job_name"))' \
              '    if name in targets:' \
              '        status = _g(job, "status")' \
              '        conclusion = _g(job, "conclusion", "result")' \
              '        runner = _g(job, "runner_name", "runner")' \
              '        started = _g(job, "started_at", "start_time")' \
              '        completed = _g(job, "completed_at", "end_time")' \
              '        print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")')"
          else
            echo "python3 unavailable; emitting raw jobs payload head"
            sed -n '1,120p' /tmp/actions-jobs.json || true
          fi

          exit 0

  integration-tests:
    name: Runtime Black-Box Integration Tests
    # Use the broader runner pool; only docker build steps are 8gb-exclusive.
    runs-on: ubuntu-act
    timeout-minutes: 20
    needs: [build_cicd, production-images-complete, build-release-images, build-tester-images]
    steps:
      - *identify_runner_step

      - *configure_registry_host_step
      - name: Run runtime black-box integration checks via compose
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
          DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }}
          DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
          INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.integration_tester_digest_ref }}
        run: |
          set -euo pipefail

          RUN_ID="${GITHUB_RUN_ID:-local}"
          RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
          LOG_FILE="$(mktemp)"
          COMPOSE_FILE="/tmp/compose.ci.integration.yaml"
          COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}"
          DB_PASSWORD="plex_password"
          DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist"
          POSTGRES_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/postgres:16-alpine"
          POSTGRES_IMAGE_UPSTREAM="docker.io/library/postgres:16-alpine"
          RESOLVER_PATH="/tmp/resolve-mirrored-image.sh"

          if ! command -v docker >/dev/null 2>&1; then
            echo "❌ Docker is required"
            exit 1
          fi
          if ! docker compose version >/dev/null 2>&1; then
            echo "❌ docker compose plugin is required"
            exit 1
          fi

          login_ok=false
          for i in 1 2 3 4 5; do
            if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
              login_ok=true
              break
            fi
            if [ "${i}" -lt 5 ]; then
              sleep_seconds=$((3 * i))
              echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
              sleep "${sleep_seconds}"
            fi
          done
          if [ "${login_ok}" != "true" ]; then
            echo "❌ Failed docker login after retries"
            exit 1
          fi

          RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
          RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"
          docker cp "${RESOLVER_CONTAINER}:/workspace/scripts/resolve-mirrored-image.sh" "${RESOLVER_PATH}"
          docker rm -f "${RESOLVER_CONTAINER}" >/dev/null 2>&1 || true
          chmod +x "${RESOLVER_PATH}"

          POSTGRES_IMAGE="$(${RESOLVER_PATH} \
            --image "${POSTGRES_IMAGE_MIRROR}" \
            --mirror-image "${POSTGRES_IMAGE_MIRROR}" \
            --upstream-image "${POSTGRES_IMAGE_UPSTREAM}" \
            --registry-user "${{ github.actor }}" \
            --registry-password-env PACKAGE_ACCESS_TOKEN)"

          cat >"${COMPOSE_FILE}" <<'EOF'
          services:
            database:
              image: ${POSTGRES_IMAGE}
              environment:
                POSTGRES_DB: plex_playlist
                POSTGRES_USER: plex_user
                POSTGRES_PASSWORD: ${DB_PASSWORD}
              healthcheck:
                test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
                interval: 5s
                timeout: 3s
                retries: 12

            backend:
              image: ${DEPLOYABLE_BACKEND_IMAGE}
              environment:
                DATABASE_URL: ${DB_URL}
                ENVIRONMENT: production
              depends_on:
                database:
                  condition: service_healthy
          EOF

          fetch_backend_endpoint() {
            endpoint_path="$1"
            output_file="$2"
            probe_tmp="$(mktemp)"

            if ! docker run --rm \
              --network "${COMPOSE_PROJECT_NAME}_default" \
              "${INTEGRATION_TESTER_DIGEST_REF}" \
              python -c "$(printf '%s\n' \
                'import sys' \
                'import urllib.error' \
                'import urllib.request' \
                'endpoint_path = sys.argv[1]' \
                'url = f"http://backend:8000{endpoint_path}"' \
                'try:' \
                '    with urllib.request.urlopen(url, timeout=2) as response:' \
                '        body = response.read().decode("utf-8", errors="replace")' \
                '        print(response.status)' \
                '        print(body)' \
                'except urllib.error.HTTPError as err:' \
                '    body = err.read().decode("utf-8", errors="replace")' \
                '    print(err.code)' \
                '    print(body)' \
                'except Exception:' \
                '    print("000")' \
                '    print("")')" \
              "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then
              : >"${output_file}"
              echo "000"
              rm -f "${probe_tmp}"
              return 0
            fi

            http_code="$(head -n 1 "${probe_tmp}")"
            tail -n +2 "${probe_tmp}" >"${output_file}"
            rm -f "${probe_tmp}"
            echo "${http_code}"
          }

          dump_failure_context() {
            echo "=== Runtime Integration Failure Context ==="
            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true
            echo "--- Compose logs (tail 200) ---"
            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true
          }

          cleanup() {
            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
          }

          trap cleanup EXIT

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "pull" ]; then
                if docker pull "${image_ref}"; then
                  return 0
                fi
              else
                return 1
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          {
            if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
              echo "❌ Missing deployable backend image references from dispatch inputs"
              exit 1
            fi

            retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
            retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3

            DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
            EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
            ACTUAL_TAG_DIGEST_REF="$({
              docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
                | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
                | head -n 1
            } || true)"

            if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then
              echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
              exit 1
            fi

            if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then
              echo "❌ Tag and digest mismatch"
              echo "expected=${EXPECTED_DIGEST_REF}"
              echo "actual=${ACTUAL_TAG_DIGEST_REF}"
              exit 1
            fi

            echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}"
            echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}"

            export DB_PASSWORD DB_URL POSTGRES_IMAGE DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}"

            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend

            db_ready=false
            for i in $(seq 1 30); do
              if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
                db_ready=true
                break
              fi
              sleep 2
            done
            if [ "${db_ready}" != "true" ]; then
              echo "❌ Database did not become ready"
              dump_failure_context
              exit 1
            fi

            backend_ready=false
            for i in $(seq 1 40); do
              backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
              if [ "${backend_running_count}" -eq 0 ]; then
                echo "❌ Backend container exited before becoming healthy"
                dump_failure_context
                exit 1
              fi

              health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
              if [ "${health_code}" = "200" ]; then
                backend_ready=true
                break
              fi
              sleep 2
            done
            if [ "${backend_ready}" != "true" ]; then
              echo "❌ Backend did not become healthy"
              cat /tmp/blackbox-health.json 2>/dev/null || true
              dump_failure_context
              exit 1
            fi

            root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)"
            if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
              echo "❌ Root endpoint validation failed"
              cat /tmp/blackbox-root.json 2>/dev/null || true
              dump_failure_context
              exit 1
            fi

            compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)"
            if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
              echo "❌ Compatibility endpoint validation failed"
              cat /tmp/blackbox-compatibility.json 2>/dev/null || true
              dump_failure_context
              exit 1
            fi

            health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
            if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then
              echo "❌ Health endpoint validation failed"
              cat /tmp/blackbox-health.json 2>/dev/null || true
              dump_failure_context
              exit 1
            fi

            echo "✅ Runtime integration checks passed"
          } 2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "❌ Runtime integration checks failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of runtime integration output ---"
            tail -n 200 "${LOG_FILE}" || true
            exit "${TEST_STATUS}"
          fi

      - *failure_diagnostics_step

  integration-tests-postmortem:
    name: Integration Tests Failure Postmortem
    runs-on: ubuntu-act
    needs: integration-tests
    if: always() && needs['integration-tests'].result == 'failure'
    timeout-minutes: 10
    steps:
      - name: Integration postmortem diagnostics
        run: |
          echo "=== Integration Tests Postmortem ==="
          echo "integration_tests_result=${{ needs['integration-tests'].result }}"
          uname -a || true
          cat /etc/os-release 2>/dev/null || true
          df -h || true
          free -h || true
          if command -v docker >/dev/null 2>&1; then
            docker version || true
            docker info || true
          fi
          timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true

  e2e-tests:
    name: End-to-End Tests
    # Use ubuntu-act runner pool for consistent availability.
    runs-on: ubuntu-act
    timeout-minutes: 45
    needs: [build_cicd, production-images-complete, build-release-images, build-tester-images]
    steps:
      - *identify_runner_step

      - *configure_registry_host_step
      - name: Run E2E tests against runtime services via compose
        env:
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
          DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }}
          DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
          DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_tag_ref }}
          DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }}
          E2E_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.e2e_tester_digest_ref }}
        run: |
          set -euo pipefail
          LOG_FILE="$(mktemp)"
          ARTIFACT_DIR="$(mktemp -d)"
          RUN_ID="${GITHUB_RUN_ID:-local}"
          RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
          COMPOSE_FILE="/tmp/compose.ci.e2e.yaml"
          COMPOSE_PROJECT_NAME="plex-e2e-${RUN_ID}-${RUN_ATTEMPT}"
          DB_PASSWORD="plex_password"
          DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist"
          E2E_REPO_DIR="$(mktemp -d)"
          POSTGRES_IMAGE_MIRROR="${GITEA_REGISTRY}/darkhelm.org/postgres:16-alpine"
          POSTGRES_IMAGE_UPSTREAM="docker.io/library/postgres:16-alpine"
          RESOLVER_PATH="/tmp/resolve-mirrored-image.sh"

          if ! command -v docker >/dev/null 2>&1; then
            echo "❌ Docker is required"
            exit 1
          fi
          if ! docker compose version >/dev/null 2>&1; then
            echo "❌ docker compose plugin is required"
            exit 1
          fi

          login_ok=false
          for i in 1 2 3 4 5; do
            if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
              login_ok=true
              break
            fi
            if [ "${i}" -lt 5 ]; then
              sleep_seconds=$((3 * i))
              echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
              sleep "${sleep_seconds}"
            fi
          done
          if [ "${login_ok}" != "true" ]; then
            echo "❌ Failed docker login after retries"
            exit 1
          fi

          RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
          RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"
          docker cp "${RESOLVER_CONTAINER}:/workspace/scripts/resolve-mirrored-image.sh" "${RESOLVER_PATH}"
          docker rm -f "${RESOLVER_CONTAINER}" >/dev/null 2>&1 || true
          chmod +x "${RESOLVER_PATH}"

          POSTGRES_IMAGE="$(${RESOLVER_PATH} \
            --image "${POSTGRES_IMAGE_MIRROR}" \
            --mirror-image "${POSTGRES_IMAGE_MIRROR}" \
            --upstream-image "${POSTGRES_IMAGE_UPSTREAM}" \
            --registry-user "${{ github.actor }}" \
            --registry-password-env PACKAGE_ACCESS_TOKEN)"

          cat >"${COMPOSE_FILE}" <<'EOF'
          services:
            database:
              image: ${POSTGRES_IMAGE}
              environment:
                POSTGRES_DB: plex_playlist
                POSTGRES_USER: plex_user
                POSTGRES_PASSWORD: ${DB_PASSWORD}
              healthcheck:
                test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
                interval: 5s
                timeout: 3s
                retries: 12

            backend:
              image: ${DEPLOYABLE_BACKEND_IMAGE}
              environment:
                DATABASE_URL: ${DB_URL}
                ENVIRONMENT: production
              depends_on:
                database:
                  condition: service_healthy

            frontend:
              image: ${DEPLOYABLE_FRONTEND_IMAGE}
              depends_on:
                backend:
                  condition: service_started
          EOF

          fetch_http() {
            container_name="$1"
            endpoint_path="$2"
            output_file="$3"
            probe_tmp="$(mktemp)"

            run_probe_with_python() {
              python_bin="$1"
              docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" "${python_bin}" -c "$(printf '%s\n' \
              'import sys' \
              'import urllib.error' \
              'import urllib.request' \
              'endpoint_path = sys.argv[1]' \
              'url = f"http://127.0.0.1:8000{endpoint_path}"' \
              'try:' \
              '    with urllib.request.urlopen(url, timeout=2) as response:' \
              '        body = response.read().decode("utf-8", errors="replace")' \
              '        print(response.status)' \
              '        print(body)' \
              'except urllib.error.HTTPError as err:' \
              '    body = err.read().decode("utf-8", errors="replace")' \
              '    print(err.code)' \
              '    print(body)' \
              'except Exception:' \
              '    print("000")' \
              '    print("")')" "${endpoint_path}"
            }

            probe_err="$(mktemp)"

            if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then
              :
            else
              : >"${output_file}"
              {
                echo '{"status":"probe_exec_failed"}'
                sed -n '1,5p' "${probe_err}" || true
              } >"${output_file}"
              rm -f "${probe_tmp}"
              rm -f "${probe_err}"
              echo "000"
              return 0
            fi

            http_code="$(head -n 1 "${probe_tmp}")"
            tail -n +2 "${probe_tmp}" >"${output_file}"
            rm -f "${probe_tmp}"
            rm -f "${probe_err}"
            echo "${http_code}"
          }

          dump_failure_context() {
            echo "=== Runtime E2E Failure Context ==="
            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true
            echo "--- Compose logs (tail 200) ---"
            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true
            echo "--- Artifact directory ---"
            find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
          }

          cleanup() {
            docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
            rm -rf "${E2E_REPO_DIR}" >/dev/null 2>&1 || true
            rm -f ~/.ssh/id_rsa >/dev/null 2>&1 || true
          }

          trap cleanup EXIT

          retry_cmd() {
            attempts="${1:-5}"
            backoff="${2:-2}"
            shift 2

            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if "$@"; then
                return 0
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Command failed (attempt ${attempt}/${attempts}): $*"
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            echo "Command failed after ${attempts} attempts: $*"
            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "pull" ]; then
                if docker pull "${image_ref}"; then
                  return 0
                fi
              else
                return 1
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
            echo "❌ Missing deployable runtime image references from dispatch inputs"
            exit 1
          fi

          SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
          SOURCE_CONTAINER="$(docker create "${SOURCE_IMAGE}")"
          docker cp "${SOURCE_CONTAINER}:/workspace/frontend" "${E2E_REPO_DIR}/frontend"
          docker rm -f "${SOURCE_CONTAINER}" >/dev/null 2>&1 || true

          if [ ! -f "${E2E_REPO_DIR}/frontend/package.json" ]; then
            echo "❌ Missing frontend/package.json in cloned E2E workspace"
            find "${E2E_REPO_DIR}" -maxdepth 3 -type f | sort || true
            exit 1
          fi

          retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
          retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3
          retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
          retry_registry_op pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" 5 3

          DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
          EXPECTED_BACKEND_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
          ACTUAL_BACKEND_TAG_DIGEST_REF="$({
            docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
              | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
              | head -n 1
          } || true)"

          if [ -z "${ACTUAL_BACKEND_TAG_DIGEST_REF}" ]; then
            echo "❌ Could not resolve backend digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
            exit 1
          fi

          if [ "${ACTUAL_BACKEND_TAG_DIGEST_REF}" != "${EXPECTED_BACKEND_DIGEST_REF}" ]; then
            echo "❌ Backend tag and digest mismatch"
            echo "expected=${EXPECTED_BACKEND_DIGEST_REF}"
            echo "actual=${ACTUAL_BACKEND_TAG_DIGEST_REF}"
            exit 1
          fi

          DEPLOYABLE_FRONTEND_REPO="${DEPLOYABLE_FRONTEND_DIGEST_REF%%@*}"
          EXPECTED_FRONTEND_DIGEST_REF="${DEPLOYABLE_FRONTEND_DIGEST_REF}"
          ACTUAL_FRONTEND_TAG_DIGEST_REF="$({
            docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" \
              | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" \
              | head -n 1
          } || true)"

          if [ -z "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" ]; then
            echo "❌ Could not resolve frontend digest from tag reference: ${DEPLOYABLE_FRONTEND_TAG_REF}"
            exit 1
          fi

          if [ "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" != "${EXPECTED_FRONTEND_DIGEST_REF}" ]; then
            echo "❌ Frontend tag and digest mismatch"
            echo "expected=${EXPECTED_FRONTEND_DIGEST_REF}"
            echo "actual=${ACTUAL_FRONTEND_TAG_DIGEST_REF}"
            exit 1
          fi

          echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}"
          echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}"

          export DB_PASSWORD DB_URL POSTGRES_IMAGE E2E_ARTIFACT_DIR="${ARTIFACT_DIR}"
          export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}"
          export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}"

          docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend

          db_ready=false
          for i in $(seq 1 30); do
            if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
              db_ready=true
              break
            fi
            sleep 2
          done
          if [ "${db_ready}" != "true" ]; then
            echo "❌ Database did not become ready"
            dump_failure_context
            exit 1
          fi

          backend_ready=false
          for i in $(seq 1 40); do
            backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
            if [ "${backend_running_count}" -eq 0 ]; then
              echo "❌ Backend container exited before becoming healthy"
              dump_failure_context
              exit 1
            fi

            health_code="$(fetch_http "backend" "/health" /tmp/e2e-backend-health.json)"
            if [ "${health_code}" = "200" ]; then
              backend_ready=true
              break
            fi
            sleep 2
          done
          if [ "${backend_ready}" != "true" ]; then
            echo "❌ Backend did not become healthy"
            cat /tmp/e2e-backend-health.json 2>/dev/null || true
            dump_failure_context
            exit 1
          fi

          root_code="$(fetch_http "backend" "/" /tmp/e2e-backend-root.json)"
          if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/e2e-backend-root.json; then
            echo "❌ Backend root endpoint validation failed"
            cat /tmp/e2e-backend-root.json 2>/dev/null || true
            dump_failure_context
            exit 1
          fi

          frontend_ready=false
          for i in $(seq 1 30); do
            frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)"
            if [ "${frontend_running_count}" -gt 0 ]; then
              if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T frontend sh -lc 'wget -q -T 2 -O /dev/null http://127.0.0.1/'; then
                frontend_ready=true
                break
              fi
            fi
            sleep 2
          done
          if [ "${frontend_ready}" != "true" ]; then
            echo "❌ Frontend did not become reachable"
            cat /tmp/e2e-frontend-root.html 2>/dev/null || true
            dump_failure_context
            exit 1
          fi

          tar -C "${E2E_REPO_DIR}" -cf - . \
            | docker run --rm -i \
              --network "${COMPOSE_PROJECT_NAME}_default" \
              -e CI=true \
              -e PLAYWRIGHT_BASE_URL="http://frontend:80" \
              -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \
              -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \
              -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \
              "${E2E_TESTER_DIGEST_REF}" \
              bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; mkdir -p /workspace; tar -xf - -C /workspace; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; find /workspace -maxdepth 3 -type f | sort | head -n 100; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \
            2>&1 | tee "${LOG_FILE}"

          TEST_STATUS=${PIPESTATUS[0]}
          if [ "${TEST_STATUS}" -ne 0 ]; then
            echo "❌ E2E tests failed (exit=${TEST_STATUS})"
            echo "--- Last 200 lines of E2E output ---"
            tail -n 200 "${LOG_FILE}" || true
            echo "--- E2E artifact files ---"
            find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
            dump_failure_context
            exit "${TEST_STATUS}"
          fi

          echo "=== E2E artifact files ==="
          find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true

      - *failure_diagnostics_step

  e2e-tests-postmortem:
    name: E2E Tests Failure Postmortem
    runs-on: ubuntu-act
    needs: e2e-tests
    if: always() && needs['e2e-tests'].result == 'failure'
    timeout-minutes: 10
    steps:
      - name: E2E postmortem diagnostics
        run: |
          echo "=== E2E Tests Postmortem ==="
          echo "e2e_tests_result=${{ needs['e2e-tests'].result }}"
          uname -a || true
          cat /etc/os-release 2>/dev/null || true
          df -h || true
          free -h || true
          if command -v docker >/dev/null 2>&1; then
            docker version || true
            docker info || true
          fi
          timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true

  promote-release-images:
    name: Promote Staging Images To Release
    runs-on: ubuntu-act
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    timeout-minutes: 15
    needs: [build_cicd, build-release-images, integration-tests, e2e-tests]
    outputs:
      release_version: ${{ steps.promote.outputs.release_version }}
      release_short_sha: ${{ steps.promote.outputs.release_short_sha }}
      release_tagged_version: ${{ steps.promote.outputs.release_tagged_version }}
      previous_release_version: ${{ steps.promote.outputs.previous_release_version }}
      backend_release_version_ref: ${{ steps.promote.outputs.backend_release_version_ref }}
      backend_release_tagged_ref: ${{ steps.promote.outputs.backend_release_tagged_ref }}
      backend_release_build_ref: ${{ steps.promote.outputs.backend_release_build_ref }}
      frontend_release_version_ref: ${{ steps.promote.outputs.frontend_release_version_ref }}
      frontend_release_tagged_ref: ${{ steps.promote.outputs.frontend_release_tagged_ref }}
      frontend_release_build_ref: ${{ steps.promote.outputs.frontend_release_build_ref }}
    steps:
      - *identify_runner_step

      - *configure_registry_host_step
      - name: Promote staging images to release tags
        id: promote
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ github.actor }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
          STAGING_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
          STAGING_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }}
        run: |
          set -euo pipefail
          umask 077
          TMP_REPO_DIR="$(mktemp -d)"
          trap 'rm -f /tmp/release_ssh_key; rm -rf "${TMP_REPO_DIR}"' EXIT

          HEAD_SHA="${HEAD_SHA:-${GITHUB_SHA:-}}"
          if [ -z "${HEAD_SHA}" ]; then
            echo "❌ Unable to resolve HEAD_SHA"
            exit 1
          fi

          retry_cmd() {
            attempts="${1:-5}"
            backoff="${2:-2}"
            shift 2

            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if "$@"; then
                return 0
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Command failed (attempt ${attempt}/${attempts}): $*"
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            echo "Command failed after ${attempts} attempts: $*"
            return 1
          }

          docker_login_with_retry() {
            attempts="${1:-5}"
            backoff="${2:-3}"
            attempt=1
            while [ "${attempt}" -le "${attempts}" ]; do
              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
                return 0
              fi
              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "docker login failed; retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done
            return 1
          }

          retry_registry_op() {
            op_name="$1"
            image_ref="$2"
            attempts="${3:-5}"
            backoff="${4:-3}"
            attempt=1

            while [ "${attempt}" -le "${attempts}" ]; do
              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
              if [ "${op_name}" = "push" ]; then
                docker_login_with_retry 3 2
                if docker push "${image_ref}"; then
                  return 0
                fi
              else
                if docker pull "${image_ref}" >/dev/null; then
                  return 0
                fi
              fi

              if [ "${attempt}" -lt "${attempts}" ]; then
                sleep_seconds=$((backoff * attempt))
                echo "Retrying in ${sleep_seconds}s"
                sleep "${sleep_seconds}"
              fi
              attempt=$((attempt + 1))
            done

            return 1
          }

          resolve_staging_digest_from_tag() {
            image_repo="$1"
            image_tag="$2"
            image_tag_ref="${image_repo}:${image_tag}"

            retry_registry_op pull "${image_tag_ref}" 5 3
            digest_ref="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${image_tag_ref}" | grep "^${image_repo}@sha256:" | head -n 1; } || true)"
            printf '%s' "${digest_ref}"
          }

          docker_login_with_retry 5 3

          if [ -z "${STAGING_BACKEND_DIGEST_REF}" ]; then
            echo "Staging backend digest output missing; deriving from staging tag ${HEAD_SHA}"
            STAGING_BACKEND_DIGEST_REF="$(resolve_staging_digest_from_tag "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-staging" "${HEAD_SHA}")"
          fi

          if [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then
            echo "Staging frontend digest output missing; deriving from staging tag ${HEAD_SHA}"
            STAGING_FRONTEND_DIGEST_REF="$(resolve_staging_digest_from_tag "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-staging" "${HEAD_SHA}")"
          fi

          if [ -z "${STAGING_BACKEND_DIGEST_REF}" ] || [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then
            echo "❌ Unable to resolve staging image digests for HEAD_SHA=${HEAD_SHA}"
            echo "backend_staging_digest_ref=${STAGING_BACKEND_DIGEST_REF:-}"
            echo "frontend_staging_digest_ref=${STAGING_FRONTEND_DIGEST_REF:-}"
            exit 1
          fi

          mkdir -p ~/.ssh
          echo "${SSH_PRIVATE_KEY}" > /tmp/release_ssh_key
          chmod 600 /tmp/release_ssh_key
          ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null || true
          GIT_SSH_COMMAND="ssh -i /tmp/release_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no"

          SHORT_SHA="$(printf '%s' "${HEAD_SHA}" | cut -c1-7)"

          HEAD_SEMVER_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
            | awk -v sha="${HEAD_SHA}" '$1==sha {tag=$2; sub("refs/tags/", "", tag); if (tag ~ /^v[0-9]+\.[0-9]+\.[0-9]+$/) print tag}' \
            | sort -V \
            | tail -n 1)"

          if [ -z "${HEAD_SEMVER_TAG}" ]; then
            ALL_SEMVER_TAGS="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
              | awk '{tag=$2; sub("refs/tags/", "", tag); if (tag ~ /^v[0-9]+\.[0-9]+\.[0-9]+$/) print tag}' \
              | sort -V)"

            LAST_SEMVER_TAG="$(printf '%s\n' "${ALL_SEMVER_TAGS}" | tail -n 1)"
            if [ -z "${LAST_SEMVER_TAG}" ]; then
              LAST_SEMVER_TAG="v0.0.0"
            fi

            TAG_MAJOR="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')"
            TAG_MINOR="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')"
            TAG_PATCH="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\3/')"
            NEXT_PATCH="$((TAG_PATCH + 1))"
            HEAD_SEMVER_TAG="v${TAG_MAJOR}.${TAG_MINOR}.${NEXT_PATCH}"

            retry_cmd 5 2 git -C "${TMP_REPO_DIR}" init -q
            retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" remote add origin "${GITEA_REPO_SSH_URL}"
            retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --no-tags --depth 200 origin "${HEAD_SHA}"
            git -C "${TMP_REPO_DIR}" config user.name "gitea-actions"
            git -C "${TMP_REPO_DIR}" config user.email "gitea-actions@darkhelm.local"
            git -C "${TMP_REPO_DIR}" tag "${HEAD_SEMVER_TAG}" "${HEAD_SHA}"
            retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" push origin "refs/tags/${HEAD_SEMVER_TAG}"
            echo "Created patch release tag ${HEAD_SEMVER_TAG} for ${HEAD_SHA}"
          fi

          TAG_MAJOR="$(printf '%s' "${HEAD_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')"
          TAG_MINOR="$(printf '%s' "${HEAD_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')"
          RELEASE_VERSION="v${TAG_MAJOR}.${TAG_MINOR}.0"
          RELEASE_TAGGED_VERSION="${HEAD_SEMVER_TAG}"
          RELEASE_BUILD_TAG="${RELEASE_TAGGED_VERSION}-${SHORT_SHA}"

          RELEASE_TAG_LIST="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
            | awk '{tag=$2; sub("refs/tags/", "", tag); if (tag ~ /^v[0-9]+\.[0-9]+\.0$/) print tag}' \
            | sort -V)"

          PREVIOUS_RELEASE_VERSION="$(printf '%s\n' "${RELEASE_TAG_LIST}" | awk -v current="${RELEASE_VERSION}" 'NF {if ($0 == current) {print prev; found=1; exit} prev=$0} END {if (!found) print prev}')"

          RELEASE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend"
          RELEASE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend"

          BACKEND_RELEASE_VERSION_REF="${RELEASE_BACKEND_REPO}:${RELEASE_VERSION}"
          BACKEND_RELEASE_TAGGED_REF="${RELEASE_BACKEND_REPO}:${RELEASE_TAGGED_VERSION}"
          BACKEND_RELEASE_BUILD_REF="${RELEASE_BACKEND_REPO}:${RELEASE_BUILD_TAG}"
          BACKEND_RELEASE_LATEST_REF="${RELEASE_BACKEND_REPO}:latest"
          FRONTEND_RELEASE_VERSION_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_VERSION}"
          FRONTEND_RELEASE_TAGGED_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_TAGGED_VERSION}"
          FRONTEND_RELEASE_BUILD_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_BUILD_TAG}"
          FRONTEND_RELEASE_LATEST_REF="${RELEASE_FRONTEND_REPO}:latest"

          if [ ! -d "${TMP_REPO_DIR}/.git" ]; then
            retry_cmd 5 2 git -C "${TMP_REPO_DIR}" init -q
            retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" remote add origin "${GITEA_REPO_SSH_URL}"
            retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --no-tags --depth 200 origin "${HEAD_SHA}"
          fi
          retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --tags --force origin

          RELEASE_NOTES_FILE="$(mktemp)"
          PREVIOUS_RELEASE_GIT_REF="${PREVIOUS_RELEASE_VERSION}"
          if [ -z "${PREVIOUS_RELEASE_GIT_REF}" ]; then
            PREVIOUS_RELEASE_VERSION="v0.0.0"
          fi

          if [ -n "${PREVIOUS_RELEASE_GIT_REF}" ]; then
            LOG_RANGE="${PREVIOUS_RELEASE_VERSION}..${HEAD_SHA}"
            CHANGES_HEADER="Changes since previous release ${PREVIOUS_RELEASE_VERSION}:"
          else
            LOG_RANGE="${HEAD_SHA}"
            CHANGES_HEADER="Changes since baseline ${PREVIOUS_RELEASE_VERSION}:"
          fi

          {
            echo "## Release ${RELEASE_VERSION}"
            echo
            echo "- Release series tag: ${RELEASE_VERSION}"
            echo "- Version tag on commit: ${RELEASE_TAGGED_VERSION}"
            echo "- Build tag: ${RELEASE_BUILD_TAG}"
            if [ -n "${PREVIOUS_RELEASE_VERSION}" ]; then
              echo "- Previous release: ${PREVIOUS_RELEASE_VERSION}"
            else
              echo "- Previous release: (none)"
            fi
            echo
            echo "${CHANGES_HEADER}"
            git -C "${TMP_REPO_DIR}" log --no-merges --pretty='- %s (%h)' "${LOG_RANGE}" || true
          } > "${RELEASE_NOTES_FILE}"

          if ! grep -q '^- ' "${RELEASE_NOTES_FILE}"; then
            {
              echo
              echo "- No commit-summary entries were found in the selected range."
            } >> "${RELEASE_NOTES_FILE}"
          fi

          {
            echo "### Generated Release Notes"
            cat "${RELEASE_NOTES_FILE}"
          } >> "$GITHUB_STEP_SUMMARY"

          docker_login_with_retry 5 3
          retry_registry_op pull "${STAGING_BACKEND_DIGEST_REF}" 5 3
          retry_registry_op pull "${STAGING_FRONTEND_DIGEST_REF}" 5 3

          BACKEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_BACKEND_DIGEST_REF}")"
          FRONTEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_FRONTEND_DIGEST_REF}")"

          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_VERSION_REF}"
          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_TAGGED_REF}"
          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_BUILD_REF}"
          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_LATEST_REF}"
          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_VERSION_REF}"
          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_TAGGED_REF}"
          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_BUILD_REF}"
          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_LATEST_REF}"

          retry_registry_op push "${BACKEND_RELEASE_VERSION_REF}" 5 4
          retry_registry_op push "${BACKEND_RELEASE_TAGGED_REF}" 5 4
          retry_registry_op push "${BACKEND_RELEASE_BUILD_REF}" 5 4
          retry_registry_op push "${BACKEND_RELEASE_LATEST_REF}" 5 4
          retry_registry_op push "${FRONTEND_RELEASE_VERSION_REF}" 5 4
          retry_registry_op push "${FRONTEND_RELEASE_TAGGED_REF}" 5 4
          retry_registry_op push "${FRONTEND_RELEASE_BUILD_REF}" 5 4
          retry_registry_op push "${FRONTEND_RELEASE_LATEST_REF}" 5 4

          echo "release_version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
          echo "release_short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
          echo "release_tagged_version=${RELEASE_TAGGED_VERSION}" >> "$GITHUB_OUTPUT"
          echo "previous_release_version=${PREVIOUS_RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
          echo "backend_release_version_ref=${BACKEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT"
          echo "backend_release_tagged_ref=${BACKEND_RELEASE_TAGGED_REF}" >> "$GITHUB_OUTPUT"
          echo "backend_release_build_ref=${BACKEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT"
          echo "frontend_release_version_ref=${FRONTEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT"
          echo "frontend_release_tagged_ref=${FRONTEND_RELEASE_TAGGED_REF}" >> "$GITHUB_OUTPUT"
          echo "frontend_release_build_ref=${FRONTEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT"

      - *failure_diagnostics_step