plex-playlist/.gitea/workflows/cicd.yml

name: Tests

on:
  workflow_dispatch:

concurrency:
  group: cicd-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  GITEA_SSH_HOST: kankali.darkhelm.lan
  GITEA_SSH_PORT: "2222"
  GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
  GITEA_REGISTRY: kankali.darkhelm.lan:3001
  GITEA_REGISTRY_IP: 10.18.75.2
  GITEA_REGISTRY_HOST: kankali.darkhelm.lan

jobs:
  changes:
    name: Detect Change Scope
    runs-on: ubuntu-act-8gb

    outputs:
      run_ci: ${{ steps.detect.outputs.run_ci }}
      run_backend: ${{ steps.detect.outputs.run_backend }}
      run_frontend: ${{ steps.detect.outputs.run_frontend }}

    steps:
      - name: Detect changed files
        id: detect
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          GITHUB_SHA: ${{ github.sha }}
          EVENT_NAME: ${{ github.event_name }}
          BEFORE_SHA: ${{ github.event.before }}
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
          set -e
          umask 077
          trap 'rm -f ~/.ssh/id_rsa' EXIT

          if [ "${EVENT_NAME}" = "workflow_dispatch" ]; then
            echo "run_ci=true" >> "$GITHUB_OUTPUT"
            echo "run_backend=true" >> "$GITHUB_OUTPUT"
            echo "run_frontend=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          if [ -z "${SSH_PRIVATE_KEY}" ]; then
            echo "❌ SSH_PRIVATE_KEY is empty"
            exit 1
          fi

          mkdir -p ~/.ssh
          echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null

          GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
            git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .

          if [ "${EVENT_NAME}" = "pull_request" ] && [ -n "${PR_BASE_SHA}" ]; then
            BASE_SHA="${PR_BASE_SHA}"
          else
            BASE_SHA="${BEFORE_SHA}"
          fi

          CHANGED_FILES=""
          if [ -z "${BASE_SHA}" ] || [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then
            GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
              git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1 || true
            CHANGED_FILES=$(git show --pretty='' --name-only "${GITHUB_SHA}" | sed '/^$/d' || true)
          else
            GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
              git fetch --depth 1 origin "${BASE_SHA}" >/dev/null 2>&1 || true
            GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
              git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1 || true
            CHANGED_FILES=$(git diff --name-only "${BASE_SHA}" "${GITHUB_SHA}" | sed '/^$/d' || true)
          fi

          echo "Changed files:"
          echo "${CHANGED_FILES:-<none>}"

          NON_DOC_FILES=$(printf '%s\n' "${CHANGED_FILES}" | grep -E -v '^(docs/|.*\.md$)' || true)
          if [ -z "${NON_DOC_FILES}" ]; then
            echo "run_ci=false" >> "$GITHUB_OUTPUT"
            echo "run_backend=false" >> "$GITHUB_OUTPUT"
            echo "run_frontend=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          only_backend=true
          only_frontend=true

          while IFS= read -r file; do
            [ -z "${file}" ] && continue
            case "${file}" in
              backend/*) only_frontend=false ;;
              frontend/*) only_backend=false ;;
              *)
                only_backend=false
                only_frontend=false
                ;;
            esac
          done <<< "${NON_DOC_FILES}"

          echo "run_ci=true" >> "$GITHUB_OUTPUT"
          if [ "${only_backend}" = "true" ]; then
            echo "run_backend=true" >> "$GITHUB_OUTPUT"
            echo "run_frontend=false" >> "$GITHUB_OUTPUT"
          elif [ "${only_frontend}" = "true" ]; then
            echo "run_backend=false" >> "$GITHUB_OUTPUT"
            echo "run_frontend=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_backend=true" >> "$GITHUB_OUTPUT"
            echo "run_frontend=true" >> "$GITHUB_OUTPUT"
          fi

  registry-preflight:
    name: Registry Push Preflight
    runs-on: ubuntu-act-8gb
    needs: changes
    if: needs.changes.outputs.run_ci == 'true'

    steps:
      - &write_registry_push_helpers
        name: Write registry push helpers
        run: |
          cat > /tmp/registry-push-helpers.sh <<'EOF'
          #!/bin/bash

          ensure_skopeo() {
            if command -v skopeo >/dev/null 2>&1; then
              return 0
            fi

            if command -v apt-get >/dev/null 2>&1; then
              apt-get update && apt-get install -y skopeo
            elif command -v apk >/dev/null 2>&1; then
              apk add --no-cache skopeo
            fi

            if ! command -v skopeo >/dev/null 2>&1; then
              echo "❌ skopeo not available for fallback push"
              return 1
            fi
          }

          push_ref_with_fallback() {
            ref="$1"
            local_image="$2"
            archive_path="$3"
            registry_user="$4"
            package_access_token="$5"

            for i in 1 2 3; do
              echo "Docker push attempt ${i}/3 for ${ref}..."
              if docker push "${ref}"; then
                echo "✓ Docker push succeeded for ${ref}"
                return 0
              fi

              if [ "${i}" -lt 3 ]; then
                sleep 10
              fi
            done

            echo "⚠ Docker push failed for ${ref}; trying skopeo fallback"

            if ! ensure_skopeo; then
              return 1
            fi

            if [ ! -f "${archive_path}" ]; then
              echo "Creating local image archive for fallback push..."
              if ! docker save "${local_image}" -o "${archive_path}"; then
                echo "❌ Failed to create docker archive for fallback push"
                return 1
              fi
            fi

            if skopeo copy \
              --dest-creds "${registry_user}:${package_access_token}" \
              --dest-tls-verify=false \
              "docker-archive:${archive_path}" \
              "docker://${ref}"; then
              echo "✓ Skopeo fallback push succeeded for ${ref}"
              return 0
            fi

            echo "❌ Skopeo fallback push failed for ${ref}"
            return 1
          }
          EOF

          chmod 700 /tmp/registry-push-helpers.sh

      - name: Verify registry login and minimal push
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          REGISTRY_USER: ${{ secrets.REGISTRY_USER || github.actor }}
          GITHUB_SHA: ${{ github.sha }}
        run: |
          echo "=== Registry Push Preflight ==="

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
            echo "❌ Registry login failed during preflight"
            exit 1
          fi

          PREFLIGHT_TAG="${GITHUB_SHA:-manual}"
          PREFLIGHT_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-ci-preflight:${PREFLIGHT_TAG}"

          cat <<'EOF' > /tmp/Dockerfile.preflight
          FROM scratch
          LABEL org.opencontainers.image.title="plex-playlist registry preflight"
          LABEL org.opencontainers.image.description="Minimal image used to verify registry push access before expensive CI steps"
          EOF

          if ! docker build -f /tmp/Dockerfile.preflight -t registry-preflight:latest /tmp; then
            echo "❌ Failed to build minimal preflight image"
            exit 1
          fi

          docker tag registry-preflight:latest "${PREFLIGHT_REF}"

          ARCHIVE_PATH="/tmp/registry-preflight.tar"
          source /tmp/registry-push-helpers.sh

          if push_ref_with_fallback "${PREFLIGHT_REF}" "registry-preflight:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}"; then
            echo "✓ Registry preflight push succeeded: ${PREFLIGHT_REF}"
          else
            echo "❌ Registry preflight push failed: ${PREFLIGHT_REF}"
            exit 1
          fi

  prepare-base-ref:
    name: Prepare CICD Base Reference
    runs-on: ubuntu-act-8gb
    needs: [changes, registry-preflight]
    if: needs.changes.outputs.run_ci == 'true'

    outputs:
      base_hash: ${{ steps.compute-base.outputs.base_hash }}
      base_image: ${{ steps.compute-base.outputs.base_image }}

    steps:
      - name: Minimal checkout for base hash inputs
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          GITHUB_SHA: ${{ github.sha }}
        run: |
          umask 077
          trap 'rm -f ~/.ssh/id_rsa' EXIT
          echo "=== Minimal Repository Checkout for Base Hash Inputs ==="

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          # Set up SSH key securely (temporary file approach)
          if [ -n "${SSH_PRIVATE_KEY}" ]; then
            mkdir -p ~/.ssh
            echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
            chmod 600 ~/.ssh/id_rsa
            echo "Loaded SSH key fingerprint: $(ssh-keygen -lf ~/.ssh/id_rsa | awk '{print $2}')"
            ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
          else
            echo "❌ SSH_PRIVATE_KEY is empty"
            exit 1
          fi

          # Clone just enough to get the base hash inputs.
          GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
            git clone --depth 1 --no-checkout \
            "${GITEA_REPO_SSH_URL}" .

          # Checkout from the exact workflow commit SHA when available.
          if [ -n "${GITHUB_SHA}" ] && \
             GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
             git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then
            git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh
            echo "✓ Checked out base hash inputs from commit ${GITHUB_SHA}"
          else
            git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh
            echo "⚠ Falling back to default branch HEAD for base hash inputs checkout"
          fi

          chmod +x scripts/compute-cicd-base-hash.sh
          rm -f ~/.ssh/id_rsa

          echo "✓ Base hash inputs ready"

      - name: Compute immutable base image reference
        id: compute-base
        run: |
          echo "=== Computing Immutable CICD Base Reference ==="

          BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
          BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"

          echo "Base hash: ${BASE_HASH}"
          echo "Base image: ${BASE_IMAGE}"
          echo "base_hash=${BASE_HASH}" >> $GITHUB_OUTPUT
          echo "base_image=${BASE_IMAGE}" >> $GITHUB_OUTPUT

  setup:
    name: Build and Push CICD Complete Image
    runs-on: ubuntu-act-8gb
    needs: [changes, prepare-base-ref]
    if: needs.changes.outputs.run_ci == 'true'

    outputs:
      run_backend: ${{ needs.changes.outputs.run_backend }}
      run_frontend: ${{ needs.changes.outputs.run_frontend }}

    steps:
      - name: Minimal checkout for Dockerfile
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          GITHUB_SHA: ${{ github.sha }}
        run: |
          umask 077
          trap 'rm -f ~/.ssh/id_rsa' EXIT
          echo "=== Minimal Repository Checkout for Complete Dockerfile ==="

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          # Set up SSH key securely (temporary file approach)
          if [ -n "${SSH_PRIVATE_KEY}" ]; then
            mkdir -p ~/.ssh
            echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
            chmod 600 ~/.ssh/id_rsa
            echo "Loaded SSH key fingerprint: $(ssh-keygen -lf ~/.ssh/id_rsa | awk '{print $2}')"
            ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
          else
            echo "❌ SSH_PRIVATE_KEY is empty"
            exit 1
          fi

          # Clone just enough to get the Dockerfiles
          GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
            git clone --depth 1 --no-checkout \
            "${GITEA_REPO_SSH_URL}" .

          # Checkout from the exact workflow commit SHA when available.
          if [ -n "${GITHUB_SHA}" ] && \
             GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
             git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then
            git checkout FETCH_HEAD -- Dockerfile.cicd
            echo "✓ Checked out Dockerfile.cicd from commit ${GITHUB_SHA}"
          else
            git checkout HEAD -- Dockerfile.cicd
            echo "⚠ Falling back to default branch HEAD for Dockerfile checkout"
          fi

          # Clean up SSH key for security
          rm -f ~/.ssh/id_rsa

          echo "✓ Dockerfile.cicd and fallback base ready for secure build"

      - name: Build and push complete CICD image
        env:
          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          GITHUB_SHA: ${{ github.sha }}
          REGISTRY_USER: ${{ secrets.REGISTRY_USER || github.actor }}
          BASE_IMAGE: ${{ needs.prepare-base-ref.outputs.base_image }}
          BASE_HASH: ${{ needs.prepare-base-ref.outputs.base_hash }}
        run: |
          umask 077
          trap 'rm -f /tmp/ssh_key' EXIT
          echo "=== Building Complete CICD Image with Secure Secrets ==="

          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi

          # Login to registry
          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin

          echo "Waiting for immutable base image: ${BASE_IMAGE}"

          verify_base_image() {
            if timeout 30 docker manifest inspect "${BASE_IMAGE}" >/dev/null 2>&1 && \
               timeout 60 docker pull "${BASE_IMAGE}" >/dev/null 2>&1; then
              actual_hash=$(docker image inspect "${BASE_IMAGE}" --format '{{ index .Config.Labels "darkhelm.cicd-base.hash" }}' 2>/dev/null || true)
              if [ "${actual_hash}" = "${BASE_HASH}" ]; then
                echo "✓ Base image verified: ${BASE_IMAGE}"
                return 0
              fi

              echo "❌ Base image hash label mismatch: expected ${BASE_HASH}, got ${actual_hash:-<missing>}"
              return 1
            fi

            return 1
          }

          max_attempts=220
          sleep_seconds=30

          for i in $(seq 1 "${max_attempts}"); do
            echo "Base availability check ${i}/${max_attempts}..."

            if verify_base_image; then
              break
            fi

            if [ "${i}" -eq "${max_attempts}" ]; then
              echo "❌ Required immutable base image is not available: ${BASE_IMAGE}"
              echo "Timed out after $((max_attempts * sleep_seconds))s waiting for base image publish."
              echo "Ensure CICD Base Image workflow is still running or has completed successfully, then rerun main CI."
              exit 1
            fi

            echo "⚠ Base image not available yet; waiting ${sleep_seconds}s for publish workflow"
            sleep "${sleep_seconds}"
          done

          # Create temporary SSH key file for BuildKit secrets
          echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
          chmod 600 /tmp/ssh_key

          # Enable Docker BuildKit for secrets support
          export DOCKER_BUILDKIT=1

          # Build complete CICD image using secure BuildKit secrets, inheriting from base
          # SSH key is mounted securely and never stored in image layers
          echo "Building complete image with base: ${BASE_IMAGE}"
          docker build -f Dockerfile.cicd \
            --secret id=ssh_private_key,src=/tmp/ssh_key \
            --build-arg GITHUB_SHA="$GITHUB_SHA" \
            --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \
            -t cicd:latest .

          # Clean up temporary SSH key file
          rm -f /tmp/ssh_key

          # Tag for Gitea container registry
          docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest"
          docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}"

          # Push to registry (may not succeed if registry has issues)
          echo "Pushing complete CICD images to registry..."
          if docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" && \
             docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}"; then
            echo "✓ Complete CICD image pushed to registry"
          else
            echo "⚠ Registry push failed, but complete image is built locally"
            echo "Other jobs will use local images"
          fi

  # Pre-commit style checks - General file formatting
  trailing-whitespace:
    name: Trailing Whitespace Check
    runs-on: ubuntu-act
    needs: setup
    steps:
      - &configure_registry_host_step
        name: Configure registry host resolution
        run: |
          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
          fi
      - &ensure_cicd_image_step
        name: Ensure CICD image is available
        run: |
          IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}"
          if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
            echo "Using cached CICD image: ${IMAGE}"
          else
            echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ secrets.REGISTRY_USER || github.actor }}" --password-stdin
            docker pull "${IMAGE}"
          fi
      - name: Check trailing whitespace with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run trailing-whitespace --all-files --show-diff-on-failure
          "

  end-of-file-fixer:
    name: End of File Check
    runs-on: ubuntu-act
    needs: setup
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Check end of file with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run end-of-file-fixer --all-files --show-diff-on-failure
          "

  check-yaml:
    name: YAML Syntax Check
    runs-on: ubuntu-act
    needs: setup
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Check YAML files with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run check-yaml --all-files --show-diff-on-failure
          "

  check-toml:
    name: TOML Syntax Check
    runs-on: ubuntu-act
    needs: setup
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Check TOML files with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run check-toml --all-files --show-diff-on-failure
          "

  mixed-line-ending:
    name: Mixed Line Ending Check
    runs-on: ubuntu-act
    needs: setup
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Check line endings with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run mixed-line-ending --all-files --show-diff-on-failure
          "

  toml-lint:
    name: TOML Formatting Check
    runs-on: ubuntu-act
    needs: setup
    steps:
      - *configure_registry_host_step
      - name: Login to Gitea Container Registry
        run: |
          echo "=== Network-Resilient Docker Registry Login ==="
          for i in 1 2 3 4 5; do
            echo "Login attempt $i/5..."
            if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | timeout 60 docker login "http://${GITEA_REGISTRY}" -u ${{ github.actor }} --password-stdin; then
              echo "✓ Login successful"
              break
            else
              if [ $i -eq 5 ]; then
                echo "❌ All login attempts failed after network timeouts"
                exit 1
              fi
              echo "⚠ Login attempt $i failed, waiting 15s before retry..."
              sleep 15
            fi
          done
      - name: Check TOML formatting with pre-commit
        run: |
          echo "=== Network-Resilient Docker Operations ==="
          IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}"

          if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
            echo "Using cached CICD image: ${IMAGE}"
          else
            # Resilient docker pull with retries
            for i in 1 2 3; do
              echo "Docker pull attempt $i/3..."
              if timeout 300 docker pull "${IMAGE}"; then
                echo "✓ Docker pull successful"
                break
              else
                if [ $i -eq 3 ]; then
                  echo "❌ All docker pull attempts failed"
                  exit 1
                fi
                echo "⚠ Docker pull attempt $i failed, waiting 20s before retry..."
                sleep 20
              fi
            done
          fi

          # Run the actual test
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run pretty-format-toml --all-files --show-diff-on-failure
          "

  # Backend Python checks
  ruff-lint:
    <<: &backend_job_defaults
      runs-on: ubuntu-act
      needs: setup
      if: needs.setup.outputs.run_backend == 'true'
    name: Ruff Linting
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run ruff linting with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run ruff --all-files --show-diff-on-failure
          "

  ruff-format:
    <<: *backend_job_defaults
    name: Ruff Format Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Check ruff formatting with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run ruff-format --all-files --show-diff-on-failure
          "

  pyright:
    <<: *backend_job_defaults
    name: Pyright Type Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run pyright type checking with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run pyright --all-files --show-diff-on-failure
          "

  darglint:
    <<: *backend_job_defaults
    name: Darglint Docstring Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run darglint docstring linting with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run darglint --all-files --show-diff-on-failure
          "

  no-docstring-types:
    <<: *backend_job_defaults
    name: No Docstring Types Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run no docstring types check with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run no-docstring-types --all-files --show-diff-on-failure
          "

  # Frontend checks
  eslint:
    <<: &frontend_job_defaults
      runs-on: ubuntu-act
      needs: setup
      if: needs.setup.outputs.run_frontend == 'true'
    name: ESLint Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run ESLint with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run eslint --all-files --show-diff-on-failure
          "

  prettier:
    <<: *frontend_job_defaults
    name: Prettier Format Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Check Prettier formatting with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run prettier --all-files --show-diff-on-failure
          "

  typescript-check:
    <<: *frontend_job_defaults
    name: TypeScript Type Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run TypeScript type checking with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run typescript-check --all-files --show-diff-on-failure
          "

  tsdoc-lint:
    <<: *frontend_job_defaults
    name: TSDoc Lint Check
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run TSDoc linting with pre-commit
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace &&
            pre-commit run tsdoc-lint --all-files --show-diff-on-failure
          "

  # Unit tests with coverage
  backend-tests:
    <<: *backend_job_defaults
    name: Backend Tests
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run backend tests with coverage
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace/backend &&
            source .venv/bin/activate &&
            uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95
          "

  frontend-tests:
    <<: *frontend_job_defaults
    name: Frontend Tests
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run frontend tests with coverage
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace/frontend &&
            yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85
          "

  # Doctest for backend
  xdoctest:
    <<: *backend_job_defaults
    name: Backend Doctests
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run backend doctests
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace/backend &&
            source .venv/bin/activate &&
            echo 'Running doctests...' &&
            if uv run xdoctest src/ --quiet; then
              echo '✓ All doctests passed'
            else
              echo 'ℹ No doctests found or some doctests failed'
              # Don't fail the build for missing doctests, only for failed ones
              if uv run xdoctest src/ --quiet --verbose 2>&1 | grep -q 'FAILED'; then
                exit 1
              fi
            fi
          "

  # Integration and E2E tests (run after unit tests complete)
  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-act
    needs: backend-tests
    if: needs.backend-tests.result == 'success'
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run integration tests
        run: |
          docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace/backend &&
            source .venv/bin/activate &&
            if [ -d 'tests/integration' ]; then
              uv run pytest tests/integration/ -v --tb=short
            else
              echo 'ℹ No integration tests found'
            fi
          "

  e2e-tests:
    name: End-to-End Tests
    runs-on: ubuntu-act
    needs: frontend-tests
    if: needs.frontend-tests.result == 'success'
    steps:
      - *configure_registry_host_step
      - *ensure_cicd_image_step
      - name: Run E2E tests
        run: |
          echo "=== Network-Resilient E2E Test Execution ==="

          # Run E2E tests with network resilience
          docker run --rm -e CI=true "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${GITHUB_SHA:-latest}" bash -c "
            cd /workspace/frontend &&
            if [ -d 'tests/e2e' ] || grep -q 'playwright' package.json; then
              echo 'Running E2E tests with Playwright (Network Resilient)...' &&
              export CI=true &&
              export NODE_ENV=test &&

              # Enhanced network-resilient Playwright setup
              echo 'Verifying Playwright installation...' &&
              yarn playwright --version &&

              echo 'Ensuring Playwright Chromium browser is available...' &&
              if timeout 300 yarn playwright install chromium; then
                echo '✓ Playwright Chromium browser is ready'
              else
                echo '❌ Failed to ensure Playwright Chromium browser'
                exit 1
              fi &&

              echo 'Running E2E tests with network resilience...' &&
              # Set additional network timeout environment variables
              export PLAYWRIGHT_TIMEOUT=90000 &&
              export NODE_TLS_REJECT_UNAUTHORIZED=0 &&
              yarn test:e2e --reporter=list --timeout=90000
            else
              echo 'ℹ No E2E tests found'
            fi
          "