diff --git a/.dockerignore b/.dockerignore index c6e9b4e..1dd3662 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,6 +25,7 @@ secrets/ .ruff_cache/ **/__pycache__/ **/.pytest_cache/ +**/.venv/ **/node_modules/ **/dist/ **/build/ diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml deleted file mode 100644 index 0b2fe3e..0000000 --- a/.gitea/workflows/cicd-source-checks.yaml +++ /dev/null @@ -1,478 +0,0 @@ -name: CICD Source Checks - -on: - workflow_dispatch: - inputs: - head_sha: - description: Commit SHA to process - required: false - source_workflow: - description: Upstream workflow name - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - base_needed: - description: Whether base rebuild is required downstream - required: false - base_hash: - description: Immutable base hash to pass downstream - required: false - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -concurrency: - group: source-checks-${{ github.sha }} - cancel-in-progress: true - -jobs: - setup: - name: Setup Source Checks Context - runs-on: ubuntu-act - timeout-minutes: 10 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - trace_id: ${{ steps.meta.outputs.trace_id }} - base_needed: ${{ steps.meta.outputs.base_needed }} - base_hash: ${{ steps.meta.outputs.base_hash }} - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - BASE_NEEDED_INPUT: ${{ github.event.inputs.base_needed }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - RESOLVED_BASE_NEEDED="${BASE_NEEDED_INPUT:-true}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-source-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Source Checks ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "base_needed=${RESOLVED_BASE_NEEDED}" - echo "base_hash=${BASE_HASH_INPUT:-deferred}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve source check metadata - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - BASE_NEEDED_INPUT: ${{ github.event.inputs.base_needed }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - RESOLVED_TRACE_ID="${TRACE_ID_INPUT:-cicd-source-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - RESOLVED_BASE_NEEDED="${BASE_NEEDED_INPUT:-true}" - RESOLVED_BASE_HASH="${BASE_HASH_INPUT:-deferred}" - - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - echo "trace_id=${RESOLVED_TRACE_ID}" >> "$GITHUB_OUTPUT" - echo "base_needed=${RESOLVED_BASE_NEEDED}" >> "$GITHUB_OUTPUT" - echo "base_hash=${RESOLVED_BASE_HASH}" >> "$GITHUB_OUTPUT" - - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true - - source-precommit-checks-backend: - name: Source Pre-commit Checks (backend) - runs-on: ubuntu-act - timeout-minutes: 40 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Checkout source snapshot - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -e - umask 022 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- . - echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" - else - git checkout HEAD -- . - echo "Falling back to default branch HEAD for source checks checkout" - fi - - test -f .pre-commit-config.yaml || { - echo "❌ Missing .pre-commit-config.yaml after checkout" - exit 1 - } - - test -f backend/pyproject.toml || { - echo "❌ Missing backend/pyproject.toml after checkout" - exit 1 - } - - test -f frontend/package.json || { - echo "❌ Missing frontend/package.json after checkout" - find . -maxdepth 3 -type f | sort | head -n 80 - exit 1 - } - - - name: Bootstrap backend toolchain - run: | - set -euo pipefail - - if ! command -v curl >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if ! command -v uv >/dev/null 2>&1; then - curl -LsSf https://astral.sh/uv/install.sh | sh - fi - export PATH="$HOME/.local/bin:$PATH" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Install backend dependencies - run: | - set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - export UV_LINK_MODE=copy - - cd backend - uv sync --dev - - - name: Run backend pre-commit source checks - env: - CI: "true" - SKIP: "eslint,prettier,typescript-check,tsdoc-lint" - run: | - set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - cd backend - uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml - - - *failure_diagnostics_step - - source-precommit-checks-frontend: - name: Source Pre-commit Checks (frontend) - runs-on: ubuntu-act - timeout-minutes: 40 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Checkout source snapshot - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -e - umask 022 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- . - echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" - else - git checkout HEAD -- . - echo "Falling back to default branch HEAD for source checks checkout" - fi - - test -f .pre-commit-config.yaml || { - echo "❌ Missing .pre-commit-config.yaml after checkout" - exit 1 - } - - test -f frontend/package.json || { - echo "❌ Missing frontend/package.json after checkout" - find . -maxdepth 3 -type f | sort | head -n 80 - exit 1 - } - - - name: Bootstrap frontend toolchain - run: | - set -euo pipefail - - if ! command -v curl >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if ! command -v uv >/dev/null 2>&1; then - curl -LsSf https://astral.sh/uv/install.sh | sh - fi - export PATH="$HOME/.local/bin:$PATH" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - if ! command -v node >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq nodejs npm - fi - if ! command -v corepack >/dev/null 2>&1; then - npm install -g corepack - fi - - corepack enable - - - name: Install frontend dependencies - run: | - set -euo pipefail - export NODE_OPTIONS="--max-old-space-size=512" - export YARN_NETWORK_CONCURRENCY=1 - cd frontend - if ! yarn install --immutable --mode=skip-build; then - echo "Yarn install failed; retrying with npm fallback for constrained runner memory" - rm -f package-lock.json - npm install --ignore-scripts --no-audit --no-fund --prefer-offline - fi - - - name: Run frontend pre-commit source checks - env: - CI: "true" - run: | - set -euo pipefail - cd frontend - corepack yarn eslint . --max-warnings=0 - corepack yarn vue-tsc --noEmit - corepack yarn prettier --check . - - - *failure_diagnostics_step - - dispatch-build: - name: Dispatch Downstream Build - runs-on: ubuntu-act - timeout-minutes: 10 - needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend] - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Dispatch downstream workflow - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - BASE_NEEDED: ${{ needs.setup.outputs.base_needed }} - BASE_HASH: ${{ needs.setup.outputs.base_hash }} - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - TRACE_ID: ${{ needs.setup.outputs.trace_id }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - - if [ "${BASE_NEEDED}" = "true" ]; then - TARGET_WORKFLOW="docker-build-base.yaml" - else - TARGET_WORKFLOW="docker-build-main.yaml" - fi - - echo "route_decision workflow=${TARGET_WORKFLOW} head_sha=${HEAD_SHA} base_needed=${BASE_NEEDED} trace_id=${TRACE_ID}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - if [ -n "${GITEA_SSH_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_IP:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - fi - - ensure_curl() { - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - echo "❌ curl is required for dispatch and could not be installed" - return 1 - } - - ensure_curl - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-source-checks" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "${TARGET_WORKFLOW}" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Source Checks" - --trace-id "${TRACE_ID}" - --base-needed "${BASE_NEEDED}" - --base-hash "${BASE_HASH}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - echo "✅ Source checks passed; dispatching ${TARGET_WORKFLOW}" - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - *failure_diagnostics_step diff --git a/.gitea/workflows/cicd-start.yaml b/.gitea/workflows/cicd-start.yaml deleted file mode 100644 index 1fa74e5..0000000 --- a/.gitea/workflows/cicd-start.yaml +++ /dev/null @@ -1,228 +0,0 @@ -name: CICD Start - -on: - push: - branches: [ '**' ] - pull_request: - branches: [ main, develop ] - workflow_dispatch: - inputs: - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -concurrency: - group: start-${{ github.ref }} - cancel-in-progress: true - -jobs: - sanity-and-routing: - name: Sanity and Base Decision - # Keep startup lightweight to avoid runner image setup stalls. - # Use the same stable self-hosted label as the downstream CICD jobs. - runs-on: ubuntu-act - timeout-minutes: 12 - outputs: - base_needed: ${{ steps.base-decision.outputs.base_needed }} - head_sha: ${{ steps.base-decision.outputs.head_sha }} - base_hash: ${{ steps.base-decision.outputs.base_hash }} - - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - REPOSITORY: ${{ github.repository }} - ACTOR: ${{ github.actor }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - SHA: ${{ github.sha }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - TRACE_ID="${TRACE_ID_INPUT:-cicd-start-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${SHA:0:8}}" - echo "=== Dispatch Audit: CICD Start ===" - echo "event_name=${EVENT_NAME}" - echo "repository=${REPOSITORY}" - echo "actor=${ACTOR}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "sha=${SHA}" - echo "trace_id=${TRACE_ID}" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Compute routing signal - id: base-decision - run: | - set -e - - # Keep this workflow container-agnostic: do not rely on checkout/actions/node. - # docker-build-base handles canonical base hash/existence checks. - echo "head_sha=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" - echo "base_hash=deferred" >> "${GITHUB_OUTPUT}" - echo "base_needed=true" >> "${GITHUB_OUTPUT}" - - echo "Routing via docker-build-base.yaml" - echo "Base hash delegated to Docker Build Base workflow" - - - name: Dispatch downstream workflow - if: steps.base-decision.outcome == 'success' && steps.base-decision.outputs.head_sha != '' && steps.base-decision.outputs.base_needed != '' - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - BASE_NEEDED: ${{ steps.base-decision.outputs.base_needed }} - HEAD_SHA: ${{ steps.base-decision.outputs.head_sha }} - BASE_HASH: ${{ steps.base-decision.outputs.base_hash }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-start-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}" - - echo "trace_id=${TRACE_ID}" - echo "target_ref=${TARGET_REF}" - - TARGET_WORKFLOW="cicd-source-checks.yaml" - - echo "route_decision workflow=${TARGET_WORKFLOW} head_sha=${HEAD_SHA} base_needed=${BASE_NEEDED} trace_id=${TRACE_ID}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - if [ -n "${GITEA_SSH_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_IP:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - fi - - ensure_curl() { - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - echo "❌ curl is required for dispatch and could not be installed" - return 1 - } - - ensure_curl - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-start" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "${TARGET_WORKFLOW}" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Start" - --trace-id "${TRACE_ID}" - --base-needed "${BASE_NEEDED}" - --base-hash "${BASE_HASH}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml deleted file mode 100644 index d29c6ad..0000000 --- a/.gitea/workflows/cicd-tests.yaml +++ /dev/null @@ -1,575 +0,0 @@ -name: CICD Tests - -on: - workflow_dispatch: - inputs: - head_sha: - description: Commit SHA to process - required: false - deployable_backend_tag_ref: - description: Deployable backend image tag reference - required: false - deployable_backend_digest_ref: - description: Deployable backend image digest reference - required: false - source_workflow: - description: Upstream workflow name - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -env: - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -concurrency: - group: tests-${{ github.sha }} - cancel-in-progress: true - -jobs: - setup: - name: Setup Tests Context - # Use the same stable runner label as the test jobs. - runs-on: ubuntu-act - timeout-minutes: 8 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }} - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Tests ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" - echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve head SHA - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT" - - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true - - backend-tests: - name: Backend Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 25 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - &configure_registry_host_step - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - &ensure_cicd_image_step - name: Ensure CICD image is available - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" - if docker image inspect "${IMAGE}" >/dev/null 2>&1; then - echo "Using cached CICD image: ${IMAGE}" - else - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - pulled=false - for i in 1 2 3; do - echo "Pull attempt ${i}/3 for ${IMAGE}" - if docker pull "${IMAGE}"; then - pulled=true - break - fi - if [ "${i}" -lt 3 ]; then - sleep_seconds=$((5 * i)) - echo "Pull failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - done - - if [ "${pulled}" != "true" ]; then - echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}" - exit 1 - fi - fi - - - name: Run backend tests with coverage - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95 - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Backend tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of backend test output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - frontend-tests: - name: Frontend Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 25 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run frontend tests with coverage - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/frontend && - yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85 - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Frontend tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of frontend test output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - xdoctest: - name: Backend Doctests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 15 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run backend doctests - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - uv run xdoctest src/ --quiet - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Backend doctests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of doctest output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - integration-tests: - name: Runtime Black-Box Integration Tests - # Pin integration tests to high-memory worker to reduce setup-stage runner churn. - runs-on: ubuntu-act-8gb - timeout-minutes: 20 - needs: [setup, backend-tests] - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - name: Run runtime black-box integration checks - env: - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }} - run: | - set -euo pipefail - set -o pipefail - - RUN_ID="${GITHUB_RUN_ID:-local}" - RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" - LOG_FILE="$(mktemp)" - NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}" - DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}" - BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}" - DB_PASSWORD="plex_password" - DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" - - fetch_backend_endpoint() { - endpoint_path="$1" - output_file="$2" - probe_tmp="$(mktemp)" - - if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \ - 'import sys' \ - 'import urllib.error' \ - 'import urllib.request' \ - 'endpoint_path = sys.argv[1]' \ - 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ - 'try:' \ - ' with urllib.request.urlopen(url, timeout=2) as response:' \ - ' body = response.read().decode("utf-8", errors="replace")' \ - ' print(response.status)' \ - ' print(body)' \ - 'except urllib.error.HTTPError as err:' \ - ' body = err.read().decode("utf-8", errors="replace")' \ - ' print(err.code)' \ - ' print(body)' \ - 'except Exception:' \ - ' print("000")' \ - ' print("")' - )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null - then - : >"${output_file}" - rm -f "${probe_tmp}" - echo "000" - return 0 - fi - - http_code="$(head -n 1 "${probe_tmp}")" - tail -n +2 "${probe_tmp}" >"${output_file}" - rm -f "${probe_tmp}" - echo "${http_code}" - } - - dump_failure_context() { - echo "=== Runtime Integration Failure Context ===" - docker ps -a || true - echo "--- Backend logs (tail 200) ---" - docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true - echo "--- Database logs (tail 200) ---" - docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true - echo "--- Backend inspect (status/image) ---" - docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true - echo "--- Database inspect (status/image) ---" - docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true - } - - cleanup() { - docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true - docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true - docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true - } - - trap cleanup EXIT - - { - if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Missing deployable backend image references from dispatch inputs" - exit 1 - fi - - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" - - DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" - EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" - ACTUAL_TAG_DIGEST_REF="$({ - docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ - | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ - | head -n 1 - } || true)" - - if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then - echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" - exit 1 - fi - - if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then - echo "❌ Tag and digest mismatch" - echo "expected=${EXPECTED_DIGEST_REF}" - echo "actual=${ACTUAL_TAG_DIGEST_REF}" - exit 1 - fi - - echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" - echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" - - docker network create "${NETWORK_NAME}" - - docker run -d \ - --name "${DB_CONTAINER}" \ - --network "${NETWORK_NAME}" \ - -e POSTGRES_DB=plex_playlist \ - -e POSTGRES_USER=plex_user \ - -e POSTGRES_PASSWORD="${DB_PASSWORD}" \ - postgres:16-alpine - - db_ready=false - for i in $(seq 1 30); do - if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then - db_ready=true - break - fi - sleep 2 - done - if [ "${db_ready}" != "true" ]; then - echo "❌ Database did not become ready" - dump_failure_context - exit 1 - fi - - docker run -d \ - --name "${BACKEND_CONTAINER}" \ - --network "${NETWORK_NAME}" \ - -p 18000:8000 \ - -e DATABASE_URL="${DB_URL}" \ - -e ENVIRONMENT=production \ - "${EXPECTED_DIGEST_REF}" - - backend_ready=false - for i in $(seq 1 40); do - backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)" - if [ "${backend_running}" != "true" ]; then - echo "❌ Backend container exited before becoming healthy" - dump_failure_context - exit 1 - fi - - health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" - if [ "${health_code}" = "200" ]; then - backend_ready=true - break - fi - sleep 2 - done - if [ "${backend_ready}" != "true" ]; then - echo "❌ Backend did not become healthy" - cat /tmp/blackbox-health.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" - if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then - echo "❌ Root endpoint validation failed" - cat /tmp/blackbox-root.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" - if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then - echo "❌ Compatibility endpoint validation failed" - cat /tmp/blackbox-compatibility.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" - if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then - echo "❌ Health endpoint validation failed" - cat /tmp/blackbox-health.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - echo "✅ Runtime black-box integration checks passed" - } 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of runtime black-box integration output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - e2e-tests: - name: End-to-End Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 30 - needs: [setup, frontend-tests] - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run E2E tests - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm -e CI=true "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/frontend && - if [ -d 'tests/e2e' ] || grep -q 'playwright' package.json; then - yarn playwright --version && - PW_BROWSER_PATH="\${PLAYWRIGHT_BROWSERS_PATH:-/root/.cache/ms-playwright}" && - if find "\${PW_BROWSER_PATH}" -maxdepth 1 -type d -name 'chromium-*' | grep -q .; then - echo 'Using preinstalled Playwright Chromium from '"\${PW_BROWSER_PATH}" - else - browser_ok=false && - for i in 1 2 3; do - echo 'Playwright browser install attempt' "\$i"'/3' && - if timeout 1800 yarn playwright install chromium; then - browser_ok=true - break - fi - if [ "\$i" -lt 3 ]; then - echo 'Playwright install attempt failed; retrying in 20s' - sleep 20 - fi - done && - if [ "\$browser_ok" != 'true' ]; then - echo '❌ Playwright browser install failed after 3 attempts' - exit 1 - fi - fi && - yarn test:e2e --reporter=list --timeout=90000 - else - echo 'No E2E tests found' - fi - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ E2E tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of E2E output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml new file mode 100644 index 0000000..c32272c --- /dev/null +++ b/.gitea/workflows/cicd.yaml @@ -0,0 +1,2432 @@ +name: CICD + +on: + push: + pull_request: + branches: [main, develop] + workflow_dispatch: + inputs: + head_sha: + description: Commit SHA to process + required: false + base_hash: + description: Immutable base hash to use for CICD base image + required: false + force_rebuild_base: + description: Force CICD base rebuild + required: false + default: "false" + trace_id: + description: Correlation id propagated across CICD jobs + required: false + +env: + GITEA_SSH_HOST: kankali.darkhelm.lan + GITEA_SSH_PORT: "2222" + GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git + GITEA_REGISTRY: kankali.darkhelm.lan:3001 + GITEA_REGISTRY_IP: 10.18.75.2 + GITEA_REGISTRY_HOST: kankali.darkhelm.lan + CI_PRUNE_AT_JOB_START: "true" + CI_PRUNE_THRESHOLD_PERCENT: "90" + +defaults: + run: + shell: bash + +concurrency: + group: cicd-${{ github.ref }} + cancel-in-progress: true + +jobs: + publish-base: + name: Build and Publish CICD Base Image + runs-on: ubuntu-act-8gb + timeout-minutes: 35 + outputs: + base_hash: ${{ steps.base-state.outputs.base_hash }} + head_sha: ${{ steps.meta.outputs.head_sha }} + steps: + - &identify_runner_step + name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - &pre_job_prune_step + name: Pre-job Docker prune + run: | + if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then + echo "Skipping pre-job prune" + exit 0 + fi + + used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" + echo "disk_used_before=${used_before}%" + echo "prune_threshold=${threshold}%" + + if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then + docker builder prune -af || true + docker system prune -af --volumes || true + used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + echo "disk_used_after=${used_after}%" + else + echo "Skipping prune; usage below threshold" + fi + + - name: Resolve head SHA + id: meta + env: + HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + HEAD_SHA_FALLBACK: ${{ github.sha }} + run: | + RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" + echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for base inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + else + git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + fi + + chmod +x scripts/compute-cicd-base-hash.sh + + - name: Compute base hash and inspect registry + id: base-state + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + FORCE_REBUILD: ${{ github.event.inputs.force_rebuild_base }} + run: | + set -euo pipefail + BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) + BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" + BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" + + echo "base_hash=${BASE_HASH}" >> "$GITHUB_OUTPUT" + echo "base_ref_hash=${BASE_REF_HASH}" >> "$GITHUB_OUTPUT" + echo "base_ref_latest=${BASE_REF_LATEST}" >> "$GITHUB_OUTPUT" + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + + if [ "${FORCE_REBUILD:-false}" = "true" ]; then + echo "needs_build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if timeout 900 docker pull "${BASE_REF_HASH}" >/dev/null 2>&1; then + echo "needs_build=false" >> "$GITHUB_OUTPUT" + else + echo "needs_build=true" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push base image + if: steps.base-state.outputs.needs_build == 'true' + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + BASE_HASH: ${{ steps.base-state.outputs.base_hash }} + BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }} + BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" + retry_registry_op pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 5 3 + + PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}") + + docker build --progress=plain -f Dockerfile.cicd-base \ + --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \ + --build-arg BASE_IMAGE_HASH="${BASE_HASH}" \ + --build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \ + -t cicd-base:latest . + + docker tag cicd-base:latest "${BASE_REF_HASH}" + docker tag cicd-base:latest "${BASE_REF_LATEST}" + retry_registry_op push "${BASE_REF_HASH}" 5 4 + retry_registry_op push "${BASE_REF_LATEST}" 5 4 + + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + fi + dmesg | tail -n 120 || true + + build_cicd: + name: Build and Push CICD Image + runs-on: ubuntu-act-8gb + needs: publish-base + timeout-minutes: 60 + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - *pre_job_prune_step + + - name: Minimal checkout for CICD build inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh + else + git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh + fi + chmod +x scripts/compute-cicd-base-hash.sh + + - name: Build and push complete CICD image + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} + BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f /tmp/ssh_key' EXIT + + trim_spaces() { + printf '%s' "$1" | tr -d '[:space:]' + } + + is_hex() { + value="$1" + if [ -z "${value}" ]; then + return 1 + fi + case "${value}" in + (*[!0-9a-fA-F]*) return 1 ;; + (*) return 0 ;; + esac + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + echo "=== Pre-build disk telemetry ===" + df -h || true + if command -v docker >/dev/null 2>&1; then + docker system df || true + docker builder prune -af || true + docker system prune -af --volumes || true + echo "=== Post-prune disk telemetry ===" + df -h || true + docker system df || true + fi + + ensure_registry_auth_realm_host + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")" + if ! is_hex "${RESOLVED_HEAD_SHA}"; then + echo "❌ Invalid or empty HEAD_SHA resolved for build_cicd: '${HEAD_SHA:-}'" + exit 1 + fi + + BASE_HASH="$(trim_spaces "${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}")" + if ! is_hex "${BASE_HASH}"; then + BASE_HASH="$(./scripts/compute-cicd-base-hash.sh | tr -d '[:space:]')" + fi + + if ! is_hex "${BASE_HASH}"; then + echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}'" + exit 1 + fi + + echo "resolved_head_sha=${RESOLVED_HEAD_SHA}" + echo "resolved_base_hash=${BASE_HASH}" + + BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" + retry_registry_op pull "${BASE_IMAGE}" 5 3 + + echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key + chmod 600 /tmp/ssh_key + + docker build -f Dockerfile.cicd \ + --secret id=ssh_private_key,src=/tmp/ssh_key \ + --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \ + --build-arg GITHUB_SHA="${RESOLVED_HEAD_SHA}" \ + --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ + -t cicd:latest . + + CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" + CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" + + docker tag cicd:latest "${CICD_LATEST_REF}" + docker tag cicd:latest "${CICD_SHA_REF}" + retry_registry_op push "${CICD_LATEST_REF}" 5 4 + retry_registry_op push "${CICD_SHA_REF}" 5 4 + + - *failure_diagnostics_step + + build-cicd-postmortem: + name: Build CICD Image Failure Postmortem + runs-on: ubuntu-act + needs: build_cicd + if: always() && needs.build_cicd.result == 'failure' + timeout-minutes: 10 + steps: + - *identify_runner_step + + - name: Build CICD postmortem diagnostics + run: | + echo "=== Build CICD Postmortem ===" + echo "build_cicd_result=${{ needs.build_cicd.result }}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true + + backend-tests: + name: Backend Tests + runs-on: ubuntu-act + timeout-minutes: 25 + needs: build_cicd + steps: + - *identify_runner_step + + - &configure_registry_host_step + name: Configure registry host resolution + run: | + if [ "${CI_PRUNE_AT_JOB_START:-false}" = "true" ] && command -v docker >/dev/null 2>&1; then + used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" + echo "disk_used_before=${used_before}%" + echo "prune_threshold=${threshold}%" + if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then + docker builder prune -af || true + docker system prune -af --volumes || true + used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + echo "disk_used_after=${used_after}%" + else + echo "Skipping prune; usage below threshold" + fi + fi + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + - &ensure_cicd_image_step + name: Ensure CICD image is available + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || github.sha }} + run: | + IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + if docker image inspect "${IMAGE}" >/dev/null 2>&1; then + echo "Using cached CICD image: ${IMAGE}" + else + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + pulled=false + for i in 1 2 3; do + echo "Pull attempt ${i}/3 for ${IMAGE}" + if docker pull "${IMAGE}"; then + pulled=true + break + fi + if [ "${i}" -lt 3 ]; then + sleep_seconds=$((5 * i)) + echo "Pull failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + + if [ "${pulled}" != "true" ]; then + echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}" + exit 1 + fi + fi + + - name: Run backend tests with coverage + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace/backend && + source .venv/bin/activate && + uv run pytest -v --tb=short -m 'not integration' --cov=src --cov-report=term-missing --cov-fail-under=95 + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Backend tests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of backend test output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + precommit-tests: + name: Pre-commit Checks + # Source-level checks should be able to use the full ubuntu-act pool. + runs-on: ubuntu-act + timeout-minutes: 30 + needs: build_cicd + steps: + - *identify_runner_step + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run pre-commit checks + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace && + if [ ! -d .git ]; then + git init /workspace + fi && + /workspace/backend/.venv/bin/pre-commit run --all-files --show-diff-on-failure --config .pre-commit-config.yaml + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Pre-commit checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of pre-commit output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + frontend-tests: + name: Frontend Tests + # Source-level checks should be able to use the full ubuntu-act pool. + runs-on: ubuntu-act + timeout-minutes: 25 + needs: build_cicd + steps: + - *identify_runner_step + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run frontend tests with coverage + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace/frontend && + yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85 + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Frontend tests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of frontend test output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + xdoctest: + name: Backend Doctests + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act + timeout-minutes: 15 + needs: build_cicd + steps: + - *identify_runner_step + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run backend doctests + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace/backend && + source .venv/bin/activate && + uv run xdoctest src/ --quiet + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Backend doctests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of doctest output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + cicd-tests-complete: + name: CICD Tests Complete + runs-on: ubuntu-act + needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] + if: always() + steps: + - name: Confirm all source lanes passed + run: | + set -euo pipefail + backend_status="${{ needs['backend-tests'].result }}" + precommit_status="${{ needs['precommit-tests'].result }}" + frontend_status="${{ needs['frontend-tests'].result }}" + xdoctest_status="${{ needs.xdoctest.result }}" + + echo "backend-tests=${backend_status}" + echo "precommit-tests=${precommit_status}" + echo "frontend-tests=${frontend_status}" + echo "xdoctest=${xdoctest_status}" + + if [ "${backend_status}" != "success" ] || [ "${precommit_status}" != "success" ] || [ "${frontend_status}" != "success" ] || [ "${xdoctest_status}" != "success" ]; then + echo "❌ One or more CICD source lanes failed" + exit 1 + fi + + echo "✅ Source checks complete" + + - *failure_diagnostics_step + + build-backend-base-image: + name: Build Backend Base Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }} + backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }} + steps: + - &resolve_head_sha_from_build_step + name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for backend base image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock + else + git checkout HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock + fi + + - name: Build and push backend base image + id: backend_base_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base" + BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.backend --target dependencies -t plex-playlist-backend-base:"${HEAD_SHA}" . + docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}" + retry_registry_op push "${BACKEND_BASE_TAG_REF}" 5 4 + retry_registry_op pull "${BACKEND_BASE_TAG_REF}" 5 3 + + BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${BACKEND_BASE_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend base image digest" + exit 1 + fi + + echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-frontend-base-image: + name: Build Frontend Base Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} + frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} + steps: + - *resolve_head_sha_from_build_step + + - name: Minimal checkout for frontend base image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml + else + git checkout HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml + fi + + - name: Build and push frontend base image + id: frontend_base_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + retry_cmd 5 3 sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin' + FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base" + FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.frontend --target deps -t plex-playlist-frontend-base:"${HEAD_SHA}" . + docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}" + retry_registry_op push "${FRONTEND_BASE_TAG_REF}" 5 4 + retry_registry_op pull "${FRONTEND_BASE_TAG_REF}" 5 3 + + FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${FRONTEND_BASE_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend base image digest" + exit 1 + fi + + echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-integration-tester-image: + name: Build Integration Tester Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} + integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} + steps: + - *resolve_head_sha_from_build_step + + - name: Minimal checkout for integration tester image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.integration-tester + else + git checkout HEAD -- Dockerfile.integration-tester + fi + + - name: Build and push integration tester image + id: integration_tester_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration" + INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.integration-tester -t plex-playlist-integration:"${HEAD_SHA}" . + docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}" + + retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4 + retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3 + + INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then + echo "❌ Unable to resolve integration tester digest" + exit 1 + fi + + echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-e2e-tester-image: + name: Build E2E Tester Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} + e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} + steps: + - *resolve_head_sha_from_build_step + + - name: Minimal checkout for E2E tester image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.e2e-tester + else + git checkout HEAD -- Dockerfile.e2e-tester + fi + + - name: Build and push E2E tester image + id: e2e_tester_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + PLAYWRIGHT_BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" + + retry_base_pull() { + image_ref="$1" + attempts="${2:-6}" + backoff="${3:-5}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "base pull attempt ${attempt}/${attempts} for ${image_ref}" + + if getent hosts "${GITEA_REGISTRY_HOST}" >/dev/null 2>&1; then + echo "${GITEA_REGISTRY_HOST} DNS resolution: ok" + else + echo "${GITEA_REGISTRY_HOST} DNS resolution: failed" + fi + + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" >/dev/null || true + + if docker pull "${image_ref}"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying base image pull in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" + E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" + + retry_base_pull "${PLAYWRIGHT_BASE_IMAGE}" 6 5 + DOCKER_BUILDKIT=0 docker build --pull=false -f Dockerfile.e2e-tester \ + --build-arg PLAYWRIGHT_BASE_IMAGE="${PLAYWRIGHT_BASE_IMAGE}" \ + -t plex-playlist-e2e:"${HEAD_SHA}" . + docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}" + + retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4 + retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3 + + E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then + echo "❌ Unable to resolve E2E tester digest" + exit 1 + fi + + echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-backend-main-image: + name: Build Backend Main Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete, build-backend-base-image] + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} + steps: + - *resolve_head_sha_from_build_step + + - name: Minimal checkout for backend main image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- \ + .dockerignore \ + Dockerfile.backend \ + backend \ + scripts/verify-deployable-image-purity.sh + else + git checkout HEAD -- \ + .dockerignore \ + Dockerfile.backend \ + backend \ + scripts/verify-deployable-image-purity.sh + fi + + - name: Build and push backend main image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + + retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend main digest" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-frontend-main-image: + name: Build Frontend Main Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete, build-frontend-base-image] + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} + deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} + steps: + - *resolve_head_sha_from_build_step + + - name: Minimal checkout for frontend main image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- \ + .dockerignore \ + Dockerfile.frontend \ + frontend \ + scripts/verify-deployable-image-purity.sh + else + git checkout HEAD -- \ + .dockerignore \ + Dockerfile.frontend \ + frontend \ + scripts/verify-deployable-image-purity.sh + fi + + - name: Build and push frontend main image + id: deployable_frontend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" + + retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + + DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend main digest" + exit 1 + fi + + echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + production-images-complete: + name: Production Images Complete + runs-on: ubuntu-act + needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + if: always() + timeout-minutes: 10 + steps: + - name: Confirm required production/tester images are complete + run: | + set -euo pipefail + backend_main_status="${{ needs['build-backend-main-image'].result }}" + frontend_main_status="${{ needs['build-frontend-main-image'].result }}" + integration_tester_status="${{ needs['build-integration-tester-image'].result }}" + e2e_tester_status="${{ needs['build-e2e-tester-image'].result }}" + + echo "build-backend-main-image=${backend_main_status}" + echo "build-frontend-main-image=${frontend_main_status}" + echo "build-integration-tester-image=${integration_tester_status}" + echo "build-e2e-tester-image=${e2e_tester_status}" + + if [ "${backend_main_status}" != "success" ] || [ "${frontend_main_status}" != "success" ] || [ "${integration_tester_status}" != "success" ] || [ "${e2e_tester_status}" != "success" ]; then + echo "❌ Production Images Complete gate failed" + exit 1 + fi + + echo "✅ Production, integration tester, and E2E tester images are complete" + + - *failure_diagnostics_step + + runtime-images-postmortem: + name: Production Image Failures Postmortem + # Run on the broader runner pool so we can still capture diagnostics when + # one of the active production image jobs fails during platform setup. + runs-on: ubuntu-act + needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + if: always() && (needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') + timeout-minutes: 10 + steps: + - *identify_runner_step + + - name: Collect production image postmortem context + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + + echo "=== Postmortem Context ===" + echo "workflow=${GITHUB_WORKFLOW:-unknown}" + echo "run_id=${GITHUB_RUN_ID:-unknown}" + echo "run_number=${GITHUB_RUN_NUMBER:-unknown}" + echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" + echo "repository=${GITHUB_REPOSITORY:-unknown}" + echo "server_url=${GITHUB_SERVER_URL:-unknown}" + echo "build_backend_main_image_result=${{ needs['build-backend-main-image'].result }}" + echo "build_frontend_main_image_result=${{ needs['build-frontend-main-image'].result }}" + echo "build_integration_tester_image_result=${{ needs['build-integration-tester-image'].result }}" + echo "build_e2e_tester_image_result=${{ needs['build-e2e-tester-image'].result }}" + + echo "=== Local Runner Telemetry (postmortem job host) ===" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + + echo "=== Registry Reachability Probe ===" + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true + + if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "No GITHUB_TOKEN available for API diagnostics" + exit 0 + fi + + API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}" + JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200" + + echo "=== Actions API Job Summary ===" + echo "jobs_url=${JOBS_URL}" + if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then + echo "Could not fetch job metadata from Actions API" + exit 0 + fi + + if command -v python3 >/dev/null 2>&1; then + python3 -c "$(printf '%s\n' \ + 'import json' \ + 'from pathlib import Path' \ + '' \ + 'path = Path("/tmp/actions-jobs.json")' \ + 'payload = json.loads(path.read_text())' \ + 'jobs = payload.get("jobs") or payload.get("data") or []' \ + '' \ + 'print("job_count=", len(jobs))' \ + '' \ + 'def _g(job, *keys):' \ + ' for key in keys:' \ + ' if key in job and job[key] not in (None, ""):' \ + ' return job[key]' \ + ' return "unknown"' \ + '' \ + 'for job in jobs:' \ + ' name = _g(job, "name", "job_name")' \ + ' status = _g(job, "status")' \ + ' conclusion = _g(job, "conclusion", "result")' \ + ' runner = _g(job, "runner_name", "runner")' \ + ' started = _g(job, "started_at", "start_time")' \ + ' completed = _g(job, "completed_at", "end_time")' \ + ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \ + '' \ + 'targets = {' \ + ' "Build Backend Main Image",' \ + ' "Build Frontend Main Image",' \ + ' "Build Integration Tester Image",' \ + ' "Build E2E Tester Image",' \ + '}' \ + '' \ + 'print("=== targeted production image jobs ===")' \ + 'for job in jobs:' \ + ' name = str(_g(job, "name", "job_name"))' \ + ' if name in targets:' \ + ' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \ + ' if key in job:' \ + ' print(f"{key}={job[key]}")' \ + ' print("---")')" + else + echo "python3 unavailable; emitting raw jobs payload head" + sed -n '1,120p' /tmp/actions-jobs.json || true + fi + + exit 0 + + source-lanes-postmortem: + name: Source Lanes Failure Postmortem + # Capture follow-up diagnostics when any source lane dies during platform + # setup before step logging is available. + runs-on: ubuntu-act + needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] + if: always() && (needs['backend-tests'].result == 'failure' || needs['precommit-tests'].result == 'failure' || needs['frontend-tests'].result == 'failure' || needs.xdoctest.result == 'failure') + timeout-minutes: 10 + steps: + - *identify_runner_step + + - name: Collect source lane postmortem context + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + + echo "=== Postmortem Context ===" + echo "workflow=${GITHUB_WORKFLOW:-unknown}" + echo "run_id=${GITHUB_RUN_ID:-unknown}" + echo "run_number=${GITHUB_RUN_NUMBER:-unknown}" + echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" + echo "repository=${GITHUB_REPOSITORY:-unknown}" + echo "server_url=${GITHUB_SERVER_URL:-unknown}" + echo "backend_tests_result=${{ needs['backend-tests'].result }}" + echo "precommit_tests_result=${{ needs['precommit-tests'].result }}" + echo "frontend_tests_result=${{ needs['frontend-tests'].result }}" + echo "xdoctest_result=${{ needs.xdoctest.result }}" + + echo "=== Local Runner Telemetry (postmortem job host) ===" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + + echo "=== Registry Reachability Probe ===" + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true + + if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "No GITHUB_TOKEN available for API diagnostics" + exit 0 + fi + + API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}" + JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200" + + echo "=== Actions API Job Summary ===" + echo "jobs_url=${JOBS_URL}" + if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then + echo "Could not fetch job metadata from Actions API" + exit 0 + fi + + if command -v python3 >/dev/null 2>&1; then + python3 -c "$(printf '%s\n' \ + 'import json' \ + 'from pathlib import Path' \ + '' \ + 'path = Path("/tmp/actions-jobs.json")' \ + 'payload = json.loads(path.read_text())' \ + 'jobs = payload.get("jobs") or payload.get("data") or []' \ + '' \ + 'print("job_count=", len(jobs))' \ + '' \ + 'def _g(job, *keys):' \ + ' for key in keys:' \ + ' if key in job and job[key] not in (None, ""):' \ + ' return job[key]' \ + ' return "unknown"' \ + '' \ + 'targets = {' \ + ' "Backend Tests",' \ + ' "Pre-commit Checks",' \ + ' "Frontend Tests",' \ + ' "Backend Doctests",' \ + '}' \ + '' \ + 'for job in jobs:' \ + ' name = str(_g(job, "name", "job_name"))' \ + ' if name in targets:' \ + ' status = _g(job, "status")' \ + ' conclusion = _g(job, "conclusion", "result")' \ + ' runner = _g(job, "runner_name", "runner")' \ + ' started = _g(job, "started_at", "start_time")' \ + ' completed = _g(job, "completed_at", "end_time")' \ + ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")')" + else + echo "python3 unavailable; emitting raw jobs payload head" + sed -n '1,120p' /tmp/actions-jobs.json || true + fi + + exit 0 + + integration-tests: + name: Runtime Black-Box Integration Tests + # Use the broader runner pool; only docker build steps are 8gb-exclusive. + runs-on: ubuntu-act + timeout-minutes: 20 + needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image] + steps: + - *identify_runner_step + + - *configure_registry_host_step + - name: Run runtime black-box integration checks via compose + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} + run: | + set -euo pipefail + set -o pipefail + + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" + LOG_FILE="$(mktemp)" + COMPOSE_FILE="/tmp/compose.ci.integration.yaml" + COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" + + if ! command -v docker >/dev/null 2>&1; then + echo "❌ Docker is required" + exit 1 + fi + if ! docker compose version >/dev/null 2>&1; then + echo "❌ docker compose plugin is required" + exit 1 + fi + + cat >"${COMPOSE_FILE}" <<'EOF' + services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE} + environment: + DATABASE_URL: ${DB_URL} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + EOF + + fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker run --rm \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + "${INTEGRATION_TESTER_DIGEST_REF}" \ + python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://backend:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")')" \ + "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then + : >"${output_file}" + echo "000" + rm -f "${probe_tmp}" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + + dump_failure_context() { + echo "=== Runtime Integration Failure Context ===" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true + echo "--- Compose logs (tail 200) ---" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true + } + + cleanup() { + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "pull" ]; then + if docker pull "${image_ref}"; then + return 0 + fi + else + return 1 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + { + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable backend image references from dispatch inputs" + exit 1 + fi + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then + echo "❌ Tag and digest mismatch" + echo "expected=${EXPECTED_DIGEST_REF}" + echo "actual=${ACTUAL_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" + echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" + + export DB_PASSWORD DB_URL DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}" + + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend + + db_ready=false + for i in $(seq 1 30); do + if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + backend_ready=false + for i in $(seq 1 40); do + backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)" + if [ "${backend_running_count}" -eq 0 ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" + if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then + echo "❌ Health endpoint validation failed" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + echo "✅ Runtime integration checks passed" + } 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Runtime integration checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of runtime integration output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + integration-tests-postmortem: + name: Integration Tests Failure Postmortem + runs-on: ubuntu-act + needs: integration-tests + if: always() && needs['integration-tests'].result == 'failure' + timeout-minutes: 10 + steps: + - name: Integration postmortem diagnostics + run: | + echo "=== Integration Tests Postmortem ===" + echo "integration_tests_result=${{ needs['integration-tests'].result }}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true + + e2e-tests: + name: End-to-End Tests + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act + timeout-minutes: 45 + needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + steps: + - *identify_runner_step + + - *configure_registry_host_step + - name: Run E2E tests against runtime services via compose + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} + DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }} + DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} + E2E_TESTER_DIGEST_REF: ${{ needs['build-e2e-tester-image'].outputs.e2e_tester_digest_ref }} + run: | + set -euo pipefail + set -o pipefail + LOG_FILE="$(mktemp)" + ARTIFACT_DIR="$(mktemp -d)" + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" + COMPOSE_FILE="/tmp/compose.ci.e2e.yaml" + COMPOSE_PROJECT_NAME="plex-e2e-${RUN_ID}-${RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" + E2E_REPO_DIR="$(mktemp -d)" + + if ! command -v docker >/dev/null 2>&1; then + echo "❌ Docker is required" + exit 1 + fi + if ! docker compose version >/dev/null 2>&1; then + echo "❌ docker compose plugin is required" + exit 1 + fi + + cat >"${COMPOSE_FILE}" <<'EOF' + services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE} + environment: + DATABASE_URL: ${DB_URL} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + + frontend: + image: ${DEPLOYABLE_FRONTEND_IMAGE} + depends_on: + backend: + condition: service_started + EOF + + fetch_http() { + container_name="$1" + endpoint_path="$2" + output_file="$3" + probe_tmp="$(mktemp)" + + run_probe_with_python() { + python_bin="$1" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" "${python_bin}" -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")')" "${endpoint_path}" + } + + probe_err="$(mktemp)" + + if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then + : + else + : >"${output_file}" + { + echo '{"status":"probe_exec_failed"}' + sed -n '1,5p' "${probe_err}" || true + } >"${output_file}" + rm -f "${probe_tmp}" + rm -f "${probe_err}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + rm -f "${probe_err}" + echo "${http_code}" + } + + dump_failure_context() { + echo "=== Runtime E2E Failure Context ===" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true + echo "--- Compose logs (tail 200) ---" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true + echo "--- Artifact directory ---" + find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true + } + + cleanup() { + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true + rm -rf "${E2E_REPO_DIR}" >/dev/null 2>&1 || true + rm -f ~/.ssh/id_rsa >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "pull" ]; then + if docker pull "${image_ref}"; then + return 0 + fi + else + return 1 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable runtime image references from dispatch inputs" + exit 1 + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" "${E2E_REPO_DIR}" + + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git -C "${E2E_REPO_DIR}" fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git -C "${E2E_REPO_DIR}" checkout FETCH_HEAD -- frontend + else + git -C "${E2E_REPO_DIR}" checkout HEAD -- frontend + fi + + if [ ! -f "${E2E_REPO_DIR}/frontend/package.json" ]; then + echo "❌ Missing frontend/package.json in cloned E2E workspace" + find "${E2E_REPO_DIR}" -maxdepth 3 -type f | sort || true + exit 1 + fi + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" 5 3 + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_BACKEND_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_BACKEND_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_BACKEND_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve backend digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_BACKEND_TAG_DIGEST_REF}" != "${EXPECTED_BACKEND_DIGEST_REF}" ]; then + echo "❌ Backend tag and digest mismatch" + echo "expected=${EXPECTED_BACKEND_DIGEST_REF}" + echo "actual=${ACTUAL_BACKEND_TAG_DIGEST_REF}" + exit 1 + fi + + DEPLOYABLE_FRONTEND_REPO="${DEPLOYABLE_FRONTEND_DIGEST_REF%%@*}" + EXPECTED_FRONTEND_DIGEST_REF="${DEPLOYABLE_FRONTEND_DIGEST_REF}" + ACTUAL_FRONTEND_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve frontend digest from tag reference: ${DEPLOYABLE_FRONTEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" != "${EXPECTED_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Frontend tag and digest mismatch" + echo "expected=${EXPECTED_FRONTEND_DIGEST_REF}" + echo "actual=${ACTUAL_FRONTEND_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}" + echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}" + + export DB_PASSWORD DB_URL E2E_ARTIFACT_DIR="${ARTIFACT_DIR}" + export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}" + export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}" + + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend + + db_ready=false + for i in $(seq 1 30); do + if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + backend_ready=false + for i in $(seq 1 40); do + backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)" + if [ "${backend_running_count}" -eq 0 ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + + health_code="$(fetch_http "backend" "/health" /tmp/e2e-backend-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/e2e-backend-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(fetch_http "backend" "/" /tmp/e2e-backend-root.json)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/e2e-backend-root.json; then + echo "❌ Backend root endpoint validation failed" + cat /tmp/e2e-backend-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + frontend_ready=false + for i in $(seq 1 30); do + frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)" + if [ "${frontend_running_count}" -gt 0 ]; then + if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T frontend sh -lc 'wget -q -T 2 -O /dev/null http://127.0.0.1/'; then + frontend_ready=true + break + fi + fi + sleep 2 + done + if [ "${frontend_ready}" != "true" ]; then + echo "❌ Frontend did not become reachable" + cat /tmp/e2e-frontend-root.html 2>/dev/null || true + dump_failure_context + exit 1 + fi + + tar -C "${E2E_REPO_DIR}" -cf - . \ + | docker run --rm -i \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + -e CI=true \ + -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ + -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ + -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ + -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ + "${E2E_TESTER_DIGEST_REF}" \ + bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; mkdir -p /workspace; tar -xf - -C /workspace; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; find /workspace -maxdepth 3 -type f | sort | head -n 100; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \ + 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ E2E tests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of E2E output ---" + tail -n 200 "${LOG_FILE}" || true + echo "--- E2E artifact files ---" + find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true + dump_failure_context + exit "${TEST_STATUS}" + fi + + echo "=== E2E artifact files ===" + find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true + + - *failure_diagnostics_step + + e2e-tests-postmortem: + name: E2E Tests Failure Postmortem + runs-on: ubuntu-act + needs: e2e-tests + if: always() && needs['e2e-tests'].result == 'failure' + timeout-minutes: 10 + steps: + - name: E2E postmortem diagnostics + run: | + echo "=== E2E Tests Postmortem ===" + echo "e2e_tests_result=${{ needs['e2e-tests'].result }}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true diff --git a/.gitea/workflows/docker-build-base.yaml b/.gitea/workflows/docker-build-base.yaml deleted file mode 100644 index 35d1738..0000000 --- a/.gitea/workflows/docker-build-base.yaml +++ /dev/null @@ -1,537 +0,0 @@ -name: Docker Build Base - -on: - workflow_dispatch: - inputs: - force_rebuild: - description: Force a rebuild even when the immutable base tag already exists - required: false - default: "false" - head_sha: - description: Commit SHA to process - required: false - source_workflow: - description: Upstream workflow name - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -concurrency: - group: base-${{ github.sha }} - cancel-in-progress: true - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -defaults: - run: - shell: bash - -jobs: - publish-base: - name: Build and Publish CICD Base Image - # Use ubuntu-act runner pool which is known to be available. - runs-on: ubuntu-act - timeout-minutes: 35 - outputs: - base_hash: ${{ steps.base-state.outputs.base_hash }} - head_sha: ${{ steps.meta.outputs.head_sha }} - - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Emit startup diagnostics - shell: sh - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TARGET_LABEL: ubuntu-act - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_SUFFIX="$(printf '%s' "${RESOLVED_HEAD_SHA}" | cut -c1-8)" - TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TRACE_SUFFIX}}" - echo "=== Base Workflow Startup Audit ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "force_rebuild=${FORCE_REBUILD}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - echo "target_runner_label=${TARGET_LABEL}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "startup_audit=ok" - - - name: Preflight Docker and registry connectivity - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - run: | - set -e - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - echo "=== Docker Preflight ===" - command -v docker - docker --version - docker info >/tmp/docker-info.log 2>&1 || (echo "❌ docker info failed" && tail -n 40 /tmp/docker-info.log && exit 1) - grep -qi "${GITEA_REGISTRY}" /tmp/docker-info.log || echo "⚠ Registry not listed in docker info insecure registries: ${GITEA_REGISTRY}" - - echo "=== Registry Login Preflight ===" - if [ -z "${PACKAGE_ACCESS_TOKEN}" ]; then - echo "❌ PACKAGE_ACCESS_TOKEN is empty" - exit 1 - fi - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/tmp/docker-login.log 2>&1 || (echo "❌ Registry login preflight failed" && tail -n 40 /tmp/docker-login.log && exit 1) - echo "registry_login_preflight=ok" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Base Image ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "force_rebuild=${FORCE_REBUILD}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve head SHA - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - - - name: Minimal checkout for base inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - GITHUB_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - echo "=== Minimal Repository Checkout for Base Inputs ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if [ -n "${SSH_PRIVATE_KEY}" ]; then - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo "Loaded SSH key fingerprint: $(ssh-keygen -lf ~/.ssh/id_rsa | awk '{print $2}')" - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - else - echo "❌ SSH_PRIVATE_KEY is empty" - exit 1 - fi - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout \ - "${GITEA_REPO_SSH_URL}" . - - if [ -n "${GITHUB_SHA}" ] && \ - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh - echo "✓ Checked out base inputs from commit ${GITHUB_SHA}" - else - git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh - echo "⚠ Falling back to default branch HEAD for base inputs checkout" - fi - - chmod +x scripts/compute-cicd-base-hash.sh - rm -f ~/.ssh/id_rsa - - - name: Compute base hash and inspect registry - id: base-state - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} - run: | - echo "=== Computing CICD Base Image Hash ===" - - BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) - BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" - - echo "Base hash: ${BASE_HASH}" - echo "base_hash=${BASE_HASH}" >> $GITHUB_OUTPUT - echo "base_ref_hash=${BASE_REF_HASH}" >> $GITHUB_OUTPUT - echo "base_ref_latest=${BASE_REF_LATEST}" >> $GITHUB_OUTPUT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null 2>&1; then - echo "❌ Registry login failed during base inspection" - exit 1 - fi - - if [ "${FORCE_REBUILD}" = "true" ]; then - echo "Manual force rebuild requested" - echo "needs_build=true" >> $GITHUB_OUTPUT - exit 0 - fi - - for i in 1 2 3; do - echo "Existing base check ${i}/3 for ${BASE_REF_HASH}..." - # Manifest inspect is unreliable on some runners; use pull as truth. - if timeout 900 docker pull "${BASE_REF_HASH}" >/tmp/base-state-pull.log 2>&1; then - echo "✓ Immutable base image already exists and is pullable: ${BASE_REF_HASH}" - echo "needs_build=false" >> $GITHUB_OUTPUT - exit 0 - else - pull_exit=$? - echo "Existing base pull check failed with exit code ${pull_exit}" - if [ "${pull_exit}" -eq 124 ]; then - echo "Existing base pull timed out after 900s" - fi - if [ -s /tmp/base-state-pull.log ]; then - tail -n 20 /tmp/base-state-pull.log || true - fi - fi - - if [ "${i}" -lt 3 ]; then - sleep 10 - fi - done - - echo "ℹ Immutable base image missing or not yet pullable: ${BASE_REF_HASH}" - echo "needs_build=true" >> $GITHUB_OUTPUT - - - name: Write registry push helpers - run: | - cat > /tmp/registry-push-helpers.sh <<'EOF' - #!/bin/bash - - ensure_skopeo() { - if command -v skopeo >/dev/null 2>&1; then - return 0 - fi - - if command -v apt-get >/dev/null 2>&1; then - apt-get update && apt-get install -y skopeo - elif command -v apk >/dev/null 2>&1; then - apk add --no-cache skopeo - fi - - if ! command -v skopeo >/dev/null 2>&1; then - echo "❌ skopeo not available for fallback push" - return 1 - fi - } - - push_ref_with_fallback() { - ref="$1" - local_image="$2" - archive_path="$3" - registry_user="$4" - package_access_token="$5" - - for i in 1 2 3; do - echo "Docker push attempt ${i}/3 for ${ref}..." - if docker push "${ref}"; then - echo "✓ Docker push succeeded for ${ref}" - return 0 - fi - - if [ "${i}" -lt 3 ]; then - sleep 10 - fi - done - - echo "⚠ Docker push failed for ${ref}; trying skopeo fallback" - - if ! ensure_skopeo; then - return 1 - fi - - if [ ! -f "${archive_path}" ]; then - echo "Creating local image archive for fallback push..." - if ! docker save "${local_image}" -o "${archive_path}"; then - echo "❌ Failed to create docker archive for fallback push" - return 1 - fi - fi - - if skopeo copy \ - --dest-creds "${registry_user}:${package_access_token}" \ - --dest-tls-verify=false \ - "docker-archive:${archive_path}" \ - "docker://${ref}"; then - echo "✓ Skopeo fallback push succeeded for ${ref}" - return 0 - fi - - echo "❌ Skopeo fallback push failed for ${ref}" - return 1 - } - EOF - - chmod 700 /tmp/registry-push-helpers.sh - - - name: Build and push base image - if: steps.base-state.outputs.needs_build == 'true' - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - BASE_HASH: ${{ steps.base-state.outputs.base_hash }} - BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }} - BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }} - run: | - echo "=== Building CICD Base Image ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - echo "❌ Registry login failed before build" - exit 1 - fi - - PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" - echo "playwright_browsers_mirror_tag=${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" - - if ! docker pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" >/tmp/playwright-mirror-pull.log 2>&1; then - echo "❌ Required internal Playwright browsers mirror image is missing: ${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" - echo "Mirror this image into your internal registry before running base builds." - tail -n 80 /tmp/playwright-mirror-pull.log || true - exit 1 - fi - - PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 2>/dev/null || true) - if [ -z "${PLAYWRIGHT_BROWSERS_IMAGE}" ]; then - echo "❌ Failed to resolve immutable digest for ${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" - exit 1 - fi - - echo "playwright_browsers_image=${PLAYWRIGHT_BROWSERS_IMAGE}" - - export DOCKER_BUILDKIT=1 - - BUILD_TIMEOUT_SECONDS=5400 - BUILD_START_EPOCH=$(date +%s) - echo "docker_build_timeout_seconds=${BUILD_TIMEOUT_SECONDS}" - echo "docker_build_started_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - timeout "${BUILD_TIMEOUT_SECONDS}" docker build --progress=plain -f Dockerfile.cicd-base \ - --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \ - --build-arg BASE_IMAGE_HASH="${BASE_HASH}" \ - --build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \ - -t cicd-base:latest . - BUILD_EXIT_CODE=$? - if [ "${BUILD_EXIT_CODE}" -ne 0 ]; then - if [ "${BUILD_EXIT_CODE}" -eq 124 ]; then - echo "❌ docker build timed out after ${BUILD_TIMEOUT_SECONDS}s" - else - echo "❌ docker build failed with exit code ${BUILD_EXIT_CODE}" - fi - exit "${BUILD_EXIT_CODE}" - fi - - BUILD_END_EPOCH=$(date +%s) - BUILD_ELAPSED=$((BUILD_END_EPOCH - BUILD_START_EPOCH)) - echo "docker_build_elapsed_seconds=${BUILD_ELAPSED}" - - ARCHIVE_PATH="/tmp/cicd-base.tar" - source /tmp/registry-push-helpers.sh - - docker tag cicd-base:latest "${BASE_REF_HASH}" - docker tag cicd-base:latest "${BASE_REF_LATEST}" - - push_ref_with_fallback "${BASE_REF_HASH}" "cicd-base:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}" - push_ref_with_fallback "${BASE_REF_LATEST}" "cicd-base:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}" - - - name: Verify published base image - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }} - run: | - echo "=== Verifying Published CICD Base Image ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - echo "❌ Registry login failed during verification" - exit 1 - fi - - for i in 1 2 3 4 5 6; do - echo "Verification attempt ${i}/6 for ${BASE_REF_HASH}..." - - if docker pull "${BASE_REF_HASH}"; then - echo "✓ Published base image is pullable: ${BASE_REF_HASH}" - exit 0 - fi - - if [ "${i}" -lt 6 ]; then - echo "⚠ Pull failed; waiting 20s before retry" - sleep 20 - fi - done - - echo "❌ Published base image could not be pulled after 6 attempts: ${BASE_REF_HASH}" - exit 1 - - - name: Dispatch main build workflow - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH: ${{ steps.base-state.outputs.base_hash }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}" - - echo "trace_id=${TRACE_ID}" - echo "target_ref=${TARGET_REF}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - - if ! command -v curl >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-base" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "docker-build-main.yaml" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Base Image" - --trace-id "${TRACE_ID}" - --base-needed "true" - --base-hash "${BASE_HASH}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml deleted file mode 100644 index bdf21c2..0000000 --- a/.gitea/workflows/docker-build-main.yaml +++ /dev/null @@ -1,481 +0,0 @@ -name: Docker Build Main - -on: - workflow_dispatch: - inputs: - head_sha: - description: Commit SHA to process - required: false - base_hash: - description: Immutable base hash to use for CICD base image - required: false - source_workflow: - description: Upstream workflow name - required: false - base_needed: - description: Whether base rebuild was required - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -defaults: - run: - shell: bash - -concurrency: - group: main-build-${{ github.sha }} - cancel-in-progress: true - -jobs: - startup-audit: - name: Main Workflow Startup Audit - # Pin startup audit to high-memory worker to avoid setup-stage runner churn. - runs-on: ubuntu-act-8gb - timeout-minutes: 5 - steps: - - name: Identify runner - shell: sh - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Emit startup diagnostics - shell: sh - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - BASE_NEEDED: ${{ github.event.inputs.base_needed }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - TARGET_LABEL: ubuntu-act-8gb - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_SUFFIX="$(printf '%s' "${RESOLVED_HEAD_SHA}" | cut -c1-8)" - TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TRACE_SUFFIX}}" - echo "=== Main Workflow Startup Audit ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "base_needed=${BASE_NEEDED}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "base_hash_input=${BASE_HASH_INPUT}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - echo "target_runner_label=${TARGET_LABEL}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "startup_audit=ok" - - - name: Check control-plane reachability - shell: sh - run: | - SERVER_URL="${GITHUB_SERVER_URL:-}" - if command -v curl >/dev/null 2>&1; then - if [ -n "${SERVER_URL}" ] && curl -fsS --connect-timeout 5 --max-time 10 "${SERVER_URL%/}/api/v1/version" >/tmp/gitea-version.json 2>/dev/null; then - echo "gitea_api_reachable=true" - cat /tmp/gitea-version.json || true - else - echo "gitea_api_reachable=false" - fi - else - echo "curl unavailable; skipping reachability check" - fi - - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true - - build: - name: Build and Push CICD Complete Image - # Pin main image build to high-memory worker to reduce setup-time failures. - runs-on: ubuntu-act-8gb - needs: startup-audit - timeout-minutes: 60 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} - - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - BASE_NEEDED: ${{ github.event.inputs.base_needed }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Main Build ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "base_needed=${BASE_NEEDED}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "base_hash_input=${BASE_HASH_INPUT}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve head SHA - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - - - name: Minimal checkout for build and verification inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -e - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1 || true - - git checkout FETCH_HEAD -- \ - .dockerignore \ - Dockerfile.backend \ - Dockerfile.frontend \ - Dockerfile.cicd \ - Dockerfile.cicd-base \ - backend \ - frontend \ - scripts/compute-cicd-base-hash.sh \ - scripts/check-dockerfile-boundaries.sh \ - scripts/verify-deployable-image-purity.sh - chmod +x scripts/compute-cicd-base-hash.sh - - - name: Verify deployable runtime boundaries - run: | - set -e - bash ./scripts/check-dockerfile-boundaries.sh - - - name: Build and verify deployable runtime image purity - env: - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -e - - docker build -f Dockerfile.backend \ - -t deployable-backend:"${HEAD_SHA}" . - - docker build -f Dockerfile.frontend \ - --target production \ - -t deployable-frontend:"${HEAD_SHA}" . - - bash ./scripts/verify-deployable-image-purity.sh \ - --image deployable-backend:"${HEAD_SHA}" \ - --profile backend - - bash ./scripts/verify-deployable-image-purity.sh \ - --image deployable-frontend:"${HEAD_SHA}" \ - --profile frontend - - - name: Push deployable backend runtime image - id: deployable_backend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - - DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" - DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" - - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" - docker push "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null - - DEPLOYABLE_BACKEND_DIGEST_REF="$({ - docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ - | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ - | head -n 1 - } || true)" - - if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve deployable backend digest reference" - exit 1 - fi - - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" - - - name: Build and push complete CICD image - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - run: | - set -e - umask 077 - trap 'rm -f /tmp/ssh_key' EXIT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - - if [ -n "${BASE_HASH_INPUT}" ]; then - BASE_HASH="${BASE_HASH_INPUT}" - echo "Using provided immutable base hash from upstream workflow dispatch: ${BASE_HASH}" - else - BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) - echo "No base_hash input provided; computed base hash locally: ${BASE_HASH}" - fi - - BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - - verify_base_image() { - # Fast path: base image already present locally. - if docker image inspect "${BASE_IMAGE}" >/dev/null 2>&1; then - return 0 - fi - - # Manifest inspect unreliable on these runners; use pull as truth. - timeout 1800 docker pull "${BASE_IMAGE}" >/tmp/base-image-pull.log 2>&1 - } - - max_attempts=2 - sleep_seconds=15 - for i in $(seq 1 "${max_attempts}"); do - echo "Base availability check ${i}/${max_attempts}..." - if verify_base_image; then - echo "✓ Base image available and hash matched: ${BASE_IMAGE}" - break - else - pull_exit=$? - echo "Base pull attempt ${i} failed with exit code ${pull_exit}" - if [ "${pull_exit}" -eq 124 ]; then - echo "Pull timed out after 1800s while downloading base image" - fi - if [ -s /tmp/base-image-pull.log ]; then - tail -n 40 /tmp/base-image-pull.log || true - fi - - if [ "${i}" -eq "${max_attempts}" ]; then - echo "❌ Required immutable base image is not available or mismatched: ${BASE_IMAGE}" - exit 1 - fi - sleep "${sleep_seconds}" - fi - done - - echo "✓ Base image ready: ${BASE_IMAGE}" - - echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key - chmod 600 /tmp/ssh_key - export DOCKER_BUILDKIT=1 - - docker build -f Dockerfile.cicd \ - --secret id=ssh_private_key,src=/tmp/ssh_key \ - --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \ - --build-arg GITHUB_SHA="${HEAD_SHA}" \ - --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ - -t cicd:latest . - - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" - - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" - - - *failure_diagnostics_step - - dispatch-tests: - name: Dispatch CICD Tests - runs-on: ubuntu-act - timeout-minutes: 15 - needs: build - if: needs.build.result == 'success' - steps: - - name: Dispatch tests workflow - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - HEAD_SHA: ${{ needs.build.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}" - - echo "trace_id=${TRACE_ID}" - echo "target_ref=${TARGET_REF}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - - ensure_curl() { - if command -v curl >/dev/null 2>&1; then - return 0 - fi - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - command -v curl >/dev/null 2>&1 - } - - ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; } - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-main" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "cicd-tests.yaml" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Main Build" - --trace-id "${TRACE_ID}" - --input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" - --input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - *failure_diagnostics_step diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 0202dd4..da978ea 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -20,213 +20,84 @@ jobs: timeout-minutes: 90 steps: - - name: Setup Node.js for Renovate + - name: Prepare Renovate container image + env: + RENOVATE_IMAGE_PRIMARY: kankali.darkhelm.lan:3001/darkhelm.org/renovate:41 + RENOVATE_IMAGE_FALLBACK: ghcr.io/renovatebot/renovate:41 + GITEA_REGISTRY_HOST: kankali.darkhelm.lan + GITEA_REGISTRY_IP: 10.18.75.2 run: | - echo "=== Setting up Node.js 24 for Renovate ===" + set -euo pipefail - # Check existing Node.js - if command -v node &> /dev/null; then - echo "Current Node.js version: $(node --version)" - fi - if command -v npm &> /dev/null; then - echo "Current npm version: $(npm --version)" + retry_cmd() { + attempts="${1:-3}" + backoff="${2:-10}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2 + echo "Retrying in ${sleep_seconds}s" >&2 + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" >&2 + return 1 + } + + echo "=== Preparing Renovate container ===" + free -h || true + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - # Aggressive cleanup of all Node.js/npm installations - echo "Performing complete Node.js cleanup..." + pick_renovate_image() { + for candidate in "${RENOVATE_IMAGE_PRIMARY}" "${RENOVATE_IMAGE_FALLBACK}"; do + echo "Trying Renovate image candidate: ${candidate}" >&2 - # Stop any Node.js processes - sudo pkill -f node || true + if docker image inspect "${candidate}" >/dev/null 2>&1; then + echo "Using cached Renovate image: ${candidate}" >&2 + printf '%s\n' "${candidate}" + return 0 + fi - # Remove all package-managed Node.js installations - sudo apt-get remove -y --purge nodejs npm node || true - sudo apt-get autoremove -y --purge || true + if retry_cmd 3 15 docker pull "${candidate}" >/dev/null; then + printf '%s\n' "${candidate}" + return 0 + fi - # Remove all manual installations and caches - sudo rm -rf /usr/local/bin/node* /usr/local/bin/npm* || true - sudo rm -rf /usr/local/lib/node* /usr/local/include/node* || true - sudo rm -rf ~/.npm ~/.nvm ~/.node* || true - sudo rm -rf /root/.npm /root/.nvm /root/.node* || true - sudo rm -rf /usr/share/nodejs || true - sudo rm -rf /etc/apt/sources.list.d/nodesource.list* || true + echo "⚠ Renovate image candidate failed: ${candidate}" >&2 + done - # Clear npm environment variables that might conflict - unset npm_config_prefix npm_config_cache npm_config_globalconfig npm_config_init_module || true + return 1 + } - echo "✓ Cleanup completed" - - # Install Node.js 24 from NodeSource with error handling - echo "Installing Node.js 24..." - - # Remove any existing NodeSource repository - sudo rm -f /etc/apt/sources.list.d/nodesource.list || true - - # Add NodeSource repository - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - - - # Install with DEBIAN_FRONTEND to avoid interactive prompts - echo "Installing Node.js package..." - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs - - # Verify and fix installation - echo "=== Verifying Node.js Installation ===" - - # Check Node.js - if command -v node &> /dev/null; then - NODE_VERSION=$(node --version) - echo "✓ Node.js installed: $NODE_VERSION" - else - echo "❌ Node.js installation failed" + if ! RENOVATE_IMAGE="$(pick_renovate_image)"; then + echo "❌ Unable to prepare Renovate image from local mirror or GHCR" + echo "Tried: ${RENOVATE_IMAGE_PRIMARY}" + echo "Tried: ${RENOVATE_IMAGE_FALLBACK}" exit 1 fi - # Check npm and fix if needed - if command -v npm &> /dev/null && npm --version &> /dev/null; then - NPM_VERSION=$(npm --version) - echo "✓ npm working: $NPM_VERSION" - else - echo "⚠️ npm not working properly, reinstalling..." + echo "Using Renovate image: ${RENOVATE_IMAGE}" - # Method 1: Try to fix npm with the bundled version - if [ -f "/usr/bin/node" ] && [ -f "/usr/lib/node_modules/npm/bin/npm-cli.js" ]; then - echo "Using bundled npm..." - sudo ln -sf /usr/lib/node_modules/npm/bin/npm-cli.js /usr/bin/npm || true - sudo chmod +x /usr/bin/npm || true - fi + docker run --rm "${RENOVATE_IMAGE}" --version - # Method 2: If that doesn't work, reinstall npm manually - if ! npm --version &> /dev/null; then - echo "Manual npm installation..." - curl -L https://www.npmjs.com/install.sh | sudo sh - fi - - # Method 3: Last resort - use npx to bootstrap npm - if ! npm --version &> /dev/null; then - echo "Using node to run npm directly..." - # Create npm wrapper script - echo '#!/bin/bash' | sudo tee /usr/bin/npm > /dev/null - echo 'exec /usr/bin/node /usr/lib/node_modules/npm/bin/npm-cli.js "$@"' | sudo tee -a /usr/bin/npm > /dev/null - sudo chmod +x /usr/bin/npm - fi - - # Final verification - if npm --version &> /dev/null; then - echo "✓ npm recovered successfully: $(npm --version)" - else - echo "❌ npm recovery failed" - exit 1 - fi - fi - - # Test npm basic functionality - echo "Testing npm functionality..." - if npm config get registry &> /dev/null; then - echo "✓ npm configuration accessible" - else - echo "⚠️ npm configuration issues, but continuing..." - fi - - # Check version compatibility for Renovate - NODE_VERSION=$(node --version | cut -d'v' -f2) - echo "=== Version Compatibility Check ===" - echo "Node.js version: $NODE_VERSION" - - if [[ $(echo "$NODE_VERSION 24.10.0" | awk '{print ($1 >= $2)}') == 1 ]]; then - echo "✅ Node.js version $NODE_VERSION meets Renovate latest requirements" - echo "RENOVATE_VERSION=latest" >> $GITHUB_ENV - else - echo "⚠️ Node.js version $NODE_VERSION - will use compatible Renovate version" - echo "RENOVATE_VERSION=40.3.2" >> $GITHUB_ENV - fi - - - name: Install Renovate - run: | - echo "=== Installing Renovate ===" - - # Set npm configuration for memory efficiency - npm config set fund false - npm config set audit false - npm config set progress false - npm config set maxsockets 3 - npm config set fetch-retry-mintimeout 2000 - npm config set fetch-retry-maxtimeout 10000 - - # Use the version determined in previous step - echo "Installing Renovate version: $RENOVATE_VERSION" - - # Check available memory - echo "Memory info:" - free -h || echo "free command not available" - - # Install with retry logic and memory-efficient options - for i in 1 2 3; do - echo "Renovate installation attempt $i/3..." - - # Clear npm cache and temp files - npm cache clean --force || true - rm -rf /tmp/npm-* || true - - # Use memory-efficient installation with longer timeout - if NODE_OPTIONS="--max-old-space-size=2048" timeout 600 npm install -g "renovate@$RENOVATE_VERSION" \ - --no-audit \ - --no-fund \ - --no-optional \ - --production \ - --maxsockets=3 \ - --fetch-timeout=30000; then - echo "✓ Renovate installation successful on attempt $i" - break - else - exit_code=$? - echo "⚠️ Renovate installation attempt $i failed (exit code: $exit_code)" - - # Check if it was killed due to memory - if [ $exit_code -eq 137 ] || [ $exit_code -eq 143 ]; then - echo " Process was killed (likely out of memory)" - fi - - if [ $i -eq 3 ]; then - echo "❌ All Renovate installation attempts failed" - echo "Debugging information:" - echo "Node.js version: $(node --version)" - echo "npm version: $(npm --version)" - echo "Memory status:" - free -h || echo "Memory info unavailable" - echo "npm config:" - npm config list || echo "npm config failed" - exit 1 - fi - echo "Waiting 30 seconds before retry..." - sleep 30 - fi - done - - # Verify Renovate installation - echo "✓ Renovate version: $(renovate --version)" - echo "✓ Renovate location: $(which renovate)" + echo "RENOVATE_IMAGE=${RENOVATE_IMAGE}" >> "$GITHUB_ENV" - name: Configure Renovate for Gitea run: | echo "=== Configuring Renovate for Gitea ===" - - # Renovate reads RENOVATE_TOKEN, RENOVATE_PLATFORM, RENOVATE_ENDPOINT, - # and RENOVATE_GIT_AUTHOR directly from the environment at runtime. - # Keeping the config file minimal avoids repeated migration warnings. - cat > renovate-config.js << 'EOF' - module.exports = { - platform: 'gitea', - endpoint: 'https://dogar.darkhelm.org', - gitAuthor: 'Renovate Bot ', - repositories: ['DarkHelm.org/plex-playlist'], - onboarding: false, - requireConfig: 'required', - extends: ['local>DarkHelm.org/plex-playlist'], - prConcurrentLimit: 3, - branchConcurrentLimit: 5, - }; - EOF - - echo "✓ Renovate configuration created" + echo "✓ Renovate runtime configuration will be passed through environment and CLI flags" - name: Run Renovate env: @@ -235,28 +106,28 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} - RENOVATE_CONFIG_FILE: renovate-config.js RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://dogar.darkhelm.org + RENOVATE_ENDPOINT_INTERNAL: http://kankali.darkhelm.lan:3001/api/v1 + RENOVATE_ENDPOINT_EXTERNAL: https://dogar.darkhelm.org/api/v1 + RENOVATE_GIT_HOST: dogar.darkhelm.org + RENOVATE_GIT_USERNAME: ${{ github.actor }} + RENOVATE_ENDPOINT_IP: 10.18.75.2 RENOVATE_ALLOW_INSECURE_TLS: "true" - LOG_LEVEL: info + RENOVATE_GIT_AUTHOR: Renovate Bot + RENOVATE_ONBOARDING: "false" + RENOVATE_REQUIRE_CONFIG: required + RENOVATE_PR_CONCURRENT_LIMIT: "3" + RENOVATE_BRANCH_CONCURRENT_LIMIT: "5" + RENOVATE_NODE_ARGS: --max-old-space-size=768 + RENOVATE_OSV_VULNERABILITY_ALERTS: "false" + LOG_LEVEL: debug run: | + set -euo pipefail + echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" - RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" - if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then - API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" - RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" - else - API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" - fi - - export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" - echo "Renovate endpoint: ${RENOVATE_ENDPOINT}" - echo "Preflight API endpoint: ${API_ENDPOINT}" - CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then echo "⚠ Renovate insecure TLS mode enabled for self-signed certificate endpoint" @@ -264,6 +135,101 @@ jobs: export NODE_TLS_REJECT_UNAUTHORIZED=0 fi + configure_endpoint() { + endpoint_input="${1%/}" + endpoint_ip_fallback="${2:-}" + + if [[ "${endpoint_input}" == */api/v1 ]]; then + API_ENDPOINT="${endpoint_input}" + BASE_ENDPOINT="${endpoint_input%/api/v1}" + else + API_ENDPOINT="${endpoint_input}/api/v1" + BASE_ENDPOINT="${endpoint_input}" + fi + + ENDPOINT_AUTHORITY="$(printf '%s' "${BASE_ENDPOINT}" | sed -E 's#^https?://([^/]+).*$#\1#')" + ENDPOINT_HOST="${ENDPOINT_AUTHORITY%%:*}" + ENDPOINT_IP="$(getent hosts "${ENDPOINT_HOST}" | awk '{print $1; exit}' || true)" + if [ -z "${ENDPOINT_IP}" ] && [ -n "${endpoint_ip_fallback}" ]; then + ENDPOINT_IP="${endpoint_ip_fallback}" + echo "Using configured Renovate endpoint IP fallback: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" + if ! grep -q "${ENDPOINT_HOST}" /etc/hosts; then + echo "${ENDPOINT_IP} ${ENDPOINT_HOST}" >> /etc/hosts + fi + fi + + DOCKER_ADD_HOST_ARG="" + if [ -n "${ENDPOINT_IP}" ]; then + DOCKER_ADD_HOST_ARG="--add-host ${ENDPOINT_HOST}:${ENDPOINT_IP}" + echo "Renovate container host mapping: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" + else + echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host" + fi + + } + + select_reachable_endpoint() { + for candidate in "${RENOVATE_ENDPOINT_INTERNAL}" "${RENOVATE_ENDPOINT_EXTERNAL}"; do + fallback_ip="" + if [ "${candidate}" = "${RENOVATE_ENDPOINT_INTERNAL}" ]; then + fallback_ip="${RENOVATE_ENDPOINT_IP:-}" + fi + + configure_endpoint "${candidate}" "${fallback_ip}" + echo "Testing Renovate endpoint candidate: ${API_ENDPOINT}" + + version_status=$(curl -sS -o /tmp/renovate-version-check.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + "${API_ENDPOINT}/version" || true) + + if [ "${version_status}" = "200" ]; then + SELECTED_ENDPOINT_INPUT="${candidate}" + VERSION_STATUS="${version_status}" + return 0 + fi + + echo "⚠ Endpoint candidate failed version preflight: ${API_ENDPOINT} (status=${version_status})" + if [ -s /tmp/renovate-version-check.json ]; then + cat /tmp/renovate-version-check.json || true + fi + done + + return 1 + } + + if ! select_reachable_endpoint; then + echo "❌ Unable to reach either configured Renovate endpoint" + echo "Tried internal: ${RENOVATE_ENDPOINT_INTERNAL}" + echo "Tried external: ${RENOVATE_ENDPOINT_EXTERNAL}" + exit 1 + fi + + # Force git clone base URL to the selected endpoint host/port. + # This avoids relying on platform-advertised clone URLs that may point to an unreachable external proxy. + export RENOVATE_GIT_URL="${BASE_ENDPOINT}/" + export RENOVATE_ENDPOINT="${BASE_ENDPOINT}" + echo "Renovate endpoint (primary): ${RENOVATE_ENDPOINT}" + echo "Renovate endpoint (fallback): ${RENOVATE_ENDPOINT_EXTERNAL}" + echo "Renovate git base URL override: ${RENOVATE_GIT_URL}" + echo "Preflight API endpoint: ${API_ENDPOINT}" + + PLATFORM_VERSION="" + if [ "${VERSION_STATUS}" = "200" ]; then + PLATFORM_VERSION=$(sed -n 's/.*"version":"\([^"]*\)".*/\1/p' /tmp/renovate-version-check.json | head -n 1) + if [ -n "${PLATFORM_VERSION}" ]; then + export RENOVATE_X_PLATFORM_VERSION="${PLATFORM_VERSION}" + echo "✓ Gitea version preflight passed: ${RENOVATE_X_PLATFORM_VERSION}" + else + echo "⚠ Gitea version preflight returned 200 but version could not be parsed" + cat /tmp/renovate-version-check.json || true + fi + else + echo "⚠ Gitea version preflight failed with status ${VERSION_STATUS}" + if [ -s /tmp/renovate-version-check.json ]; then + cat /tmp/renovate-version-check.json || true + fi + fi + select_token_with_repo_access() { for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do candidate_value="${!candidate_name:-}" @@ -286,7 +252,10 @@ jobs: -H "Authorization: token ${candidate_value}" \ "${API_ENDPOINT}/orgs/${TARGET_ORG}" || true) - if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then + if [ "${REPO_STATUS}" = "200" ]; then + if [ "${USER_STATUS}" != "200" ] || [ "${ORG_STATUS}" != "200" ]; then + echo "⚠ Token candidate ${candidate_name} has repo access but limited user/org scopes (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})" + fi echo "${candidate_name}:${candidate_value}" return 0 fi @@ -299,7 +268,7 @@ jobs: if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then echo "❌ No token available for Renovate authentication with repository access" - echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." + echo "Configure RENOVATE_TOKEN with at least repository read/write access; user/org read scopes are optional for this workflow." echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}" if [ -s /tmp/renovate-auth-check-user.json ]; then echo "Last user endpoint response body:" @@ -319,10 +288,37 @@ jobs: SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" + export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" + # Rewrite external clone URLs to local-network git transport. + export RENOVATE_GIT_REWRITE_FROM="https://dogar.darkhelm.org/" + export RENOVATE_GIT_REWRITE_TO="http://${RENOVATE_GIT_USERNAME}:${SELECTED_TOKEN}@kankali.darkhelm.lan:3001/" + + # Prefer local SSH for git on kankali:2222 when keys are available in the container. + if docker run --rm ${DOCKER_ADD_HOST_ARG} \ + --entrypoint /bin/sh \ + "${RENOVATE_IMAGE}" \ + -lc 'GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5" git ls-remote --heads "ssh://git@kankali.darkhelm.lan:2222/'"${TARGET_REPO}"'.git" >/dev/null 2>&1'; then + export RENOVATE_GIT_REWRITE_TO="ssh://git@kankali.darkhelm.lan:2222/" + echo "✓ Renovate git rewrite transport: local SSH (kankali:2222)" + else + echo "⚠ Local SSH git auth unavailable in Renovate container; using internal HTTP token fallback" + fi unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" + echo "=== Renovate container connectivity preflight ===" + # Prove whether the same image/container path Renovate uses can reach Gitea. + # If this fails, the problem is container network/TLS resolution, not Renovate config. + # shellcheck disable=SC2086 + docker run --rm ${DOCKER_ADD_HOST_ARG} \ + -e API_ENDPOINT="${API_ENDPOINT}" \ + -e TARGET_REPO="${TARGET_REPO}" \ + -e RENOVATE_TOKEN \ + -e NODE_TLS_REJECT_UNAUTHORIZED \ + "${RENOVATE_IMAGE}" \ + node -e 'const endpoints = [["version", `${process.env.API_ENDPOINT}/version`, {}], ["repo", `${process.env.API_ENDPOINT}/repos/${process.env.TARGET_REPO}`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` } }], ["user", `${process.env.API_ENDPOINT}/user`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` }, optional: true }]]; let failed = false; (async () => { for (const [name, url, rawOptions] of endpoints) { const optional = Boolean(rawOptions.optional); const options = { ...rawOptions }; delete options.optional; try { const response = await fetch(url, options); console.log(`container_${name}_status=${response.status}`); if (!response.ok && !optional) { failed = true; console.log(`container_${name}_body_start`); console.log((await response.text()).slice(0, 1000)); console.log(`container_${name}_body_end`); } else if (!response.ok && optional) { console.log(`container_${name}_optional_scope_missing=true`); } } catch (error) { if (!optional) { failed = true; console.log(`container_${name}_error=${error instanceof Error ? error.message : String(error)}`); } else { console.log(`container_${name}_optional_probe_error=${error instanceof Error ? error.message : String(error)}`); } } } if (failed) { process.exit(1); } })();' + # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then export RENOVATE_DRY_RUN="full" @@ -331,7 +327,58 @@ jobs: unset RENOVATE_DRY_RUN fi - renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist + run_renovate_with_endpoint() { + endpoint="$1" + export RENOVATE_ENDPOINT="${endpoint}" + echo "Running Renovate with endpoint: ${RENOVATE_ENDPOINT}" + + # shellcheck disable=SC2086 + docker run --rm ${DOCKER_ADD_HOST_ARG} \ + -e LOG_LEVEL \ + -e NODE_TLS_REJECT_UNAUTHORIZED \ + -e GIT_SSL_NO_VERIFY=1 \ + -e GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no' \ + -e RENOVATE_TOKEN \ + -e RENOVATE_GIT_USERNAME \ + -e RENOVATE_GIT_REWRITE_FROM \ + -e RENOVATE_GIT_REWRITE_TO \ + -e RENOVATE_GIT_URL \ + -e RENOVATE_HOST_RULES \ + -e RENOVATE_X_PLATFORM_VERSION \ + -e RENOVATE_PLATFORM \ + -e RENOVATE_ENDPOINT \ + -e RENOVATE_DRY_RUN \ + -e RENOVATE_GIT_AUTHOR \ + -e RENOVATE_ONBOARDING \ + -e RENOVATE_REQUIRE_CONFIG \ + -e RENOVATE_PR_CONCURRENT_LIMIT \ + -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ + -e RENOVATE_NODE_ARGS \ + -e RENOVATE_OSV_VULNERABILITY_ALERTS \ + --entrypoint /bin/sh \ + "${RENOVATE_IMAGE}" \ + -lc 'set -e + git config --global "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "http://dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://${RENOVATE_GIT_USERNAME}@dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://${RENOVATE_TOKEN}@dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}" + echo "Configured git dogar->kankali rewrite rules" + renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' + } + + if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then + : + else + primary_status=$? + if [ "${primary_status}" -eq 137 ] || grep -qE '(^|[[:space:]])Killed($|[[:space:]])|exitcode '\''137'\''' /tmp/renovate.log; then + echo "❌ Renovate terminated by OOM/kill signal on primary endpoint; skipping endpoint retry" + exit "${primary_status}" + fi + echo "⚠ Renovate run failed with primary endpoint. Retrying with API endpoint form..." + run_renovate_with_endpoint "${API_ENDPOINT}" 2>&1 | tee /tmp/renovate.log + fi echo "✓ Renovate execution completed" diff --git a/Dockerfile.backend b/Dockerfile.backend index fdc12fa..12a6015 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -1,37 +1,30 @@ # Backend Dockerfile for FastAPI with Python 3.14 -FROM python:3.14-slim +FROM python:3.14-slim AS dependencies -# Set working directory WORKDIR /app -# Install system dependencies -RUN apt-get update && apt-get install -y \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv -# Create and activate virtual environment ENV VIRTUAL_ENV=/app/.venv -RUN uv venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# Copy dependency files first for better caching COPY backend/pyproject.toml backend/uv.lock* ./ # Hatchling resolves the readme path ../README.md relative to the backend # package root (/app). Create a stub so metadata validation succeeds without # requiring the file to pass through .dockerignore. RUN echo '# plex-playlist' > /README.md +RUN uv venv "$VIRTUAL_ENV" && uv sync --frozen --no-dev -# Install runtime dependencies only -RUN uv sync --frozen --no-dev +FROM python:3.14-slim AS runtime -# Copy application code -COPY backend/ . +WORKDIR /app + +ENV VIRTUAL_ENV=/app/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY --from=dependencies /app/.venv /app/.venv +COPY backend/src ./src -# Expose port EXPOSE 8000 -# Default command - can be overridden in compose for development CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Dockerfile.e2e-tester b/Dockerfile.e2e-tester new file mode 100644 index 0000000..a79f3ea --- /dev/null +++ b/Dockerfile.e2e-tester @@ -0,0 +1,8 @@ +ARG PLAYWRIGHT_BASE_IMAGE=mcr.microsoft/playwright:v1.56.1-jammy +FROM ${PLAYWRIGHT_BASE_IMAGE} + +WORKDIR /workspace/frontend + +RUN corepack enable + +CMD ["bash"] diff --git a/Dockerfile.frontend b/Dockerfile.frontend index be5d130..a58a56c 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -1,40 +1,38 @@ # Frontend Dockerfile for Vue/Vite TypeScript project -FROM node:20-alpine AS base +FROM node:24-alpine AS deps -# Set working directory WORKDIR /app -# Copy package files first for better caching -COPY frontend/package*.json ./ -COPY frontend/.yarnrc.yml ./ -COPY frontend/yarn.lock* frontend/pnpm-lock.yaml* ./ +COPY frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml ./ +RUN set -eux; \ + retry() { \ + attempts="$1"; shift; \ + n=1; \ + while [ "$n" -le "$attempts" ]; do \ + if "$@"; then \ + return 0; \ + fi; \ + if [ "$n" -lt "$attempts" ]; then \ + sleep "$((3 * n))"; \ + fi; \ + n=$((n + 1)); \ + done; \ + return 1; \ + }; \ + corepack enable; \ + retry 5 corepack prepare yarn@4.10.3 --activate; \ + retry 5 yarn install --immutable -# Install dependencies -RUN if [ -f yarn.lock ]; then corepack enable && corepack yarn install; \ - elif [ -f pnpm-lock.yaml ]; then npm install -g pnpm && pnpm install; \ - else npm install; fi +FROM deps AS build -# Copy application code COPY frontend/ . +RUN yarn build -# Development stage -FROM base AS development -EXPOSE 5173 -CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] - -# Production build stage -FROM base AS build -RUN if [ -f yarn.lock ]; then corepack yarn build; \ - elif [ -f pnpm-lock.yaml ]; then pnpm build; \ - else npm run build; fi - -# Production stage FROM nginx:alpine AS production -# Copy built assets from build stage + COPY --from=build /app/dist /usr/share/nginx/html -# Copy nginx configuration COPY frontend/nginx.conf /etc/nginx/nginx.conf -# Expose port + EXPOSE 80 -# Start nginx + CMD ["nginx", "-g", "daemon off;"] diff --git a/Dockerfile.integration-tester b/Dockerfile.integration-tester new file mode 100644 index 0000000..989ef9d --- /dev/null +++ b/Dockerfile.integration-tester @@ -0,0 +1,9 @@ +FROM python:3.14-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +CMD ["bash"] diff --git a/README.md b/README.md index eac02b2..d9eb8b1 100644 --- a/README.md +++ b/README.md @@ -157,9 +157,14 @@ plex-playlist/ For Gitea Actions runner image mirror maintenance, use: - `source scripts/gitea-actions/repair_runner_mirror.xsh` -- `source scripts/gitea-actions/check_runner_images.xsh` +- `source scripts/gitea-actions/check_runner_images.xsh` (checks ubuntu:22.04, GHCR act image, local mirror, and Renovate image) - `source scripts/gitea-actions/collect_runner_diagnostics.xsh [since] [trace_id]` +For setup-stage failures with sparse logs, use: + +- `source scripts/gitea-actions/runner-prechange-capture.xsh [tail_lines]` +- `source scripts/gitea-actions/diagnose_runner_startup.xsh` + For full troubleshooting context, see `docs/GITEA_ACTIONS_TROUBLESHOOTING.md`. ## Environment Variables diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 917c238..8204bac 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -154,6 +154,8 @@ reset = [ {shell = "cd frontend && rm -rf node_modules 2>/dev/null || true"}, "deps-install" ] +runner-diagnostics = {shell = "xonsh ./scripts/gitea-actions/collect_runner_diagnostics.xsh", help = "Collect recent runner, docker, and system diagnostics from all known runner hosts"} +runner-discover = {shell = "xonsh ./scripts/gitea-actions/discover-runners.xsh --batch", help = "Collect runner inventory, labels, and baseline health from all known runner hosts"} # === Development Setup (New Developer Onboarding) === setup = [ "deps-install", @@ -166,15 +168,15 @@ setup = [ ] test-all = ["test-backend", "test-frontend", "test-integration"] # === Testing Tasks === -test-backend = {shell = "cd backend && uv run pytest", help = "Run backend unit tests"} -test-backend-cov = {shell = "cd backend && uv run pytest --cov", help = "Run backend tests with coverage"} -test-e2e = {shell = "cd frontend && yarn test:e2e", help = "Run end-to-end tests"} +test-backend = {shell = "cd backend && uv run pytest -m 'not integration'", help = "Run backend unit tests (excludes integration marker)"} +test-backend-cov = {shell = "cd backend && uv run pytest -m 'not integration' --cov", help = "Run backend unit tests with coverage (excludes integration marker)"} +test-e2e = {shell = "bash ./scripts/run-ci-e2e-compose.sh", help = "Run E2E tests via compose against deployable runtime images"} test-frontend = {shell = "cd frontend && yarn test", help = "Run frontend unit tests"} test-frontend-cov = {shell = "cd frontend && yarn test:coverage", help = "Run frontend tests with coverage"} test-full = ["test-backend-cov", "test-frontend-cov", "test-integration", "test-e2e"] # === Smart Conditional Tasks === test-if-changed = {shell = "if git diff --quiet HEAD~1 backend/ frontend/; then echo 'No changes detected, skipping tests'; else poe test-unit; fi", help = "Only run tests if code has changed"} -test-integration = {shell = "cd backend && uv run pytest tests/integration/", help = "Run backend integration tests"} +test-integration = {shell = "bash ./scripts/run-ci-integration-compose.sh", help = "Run integration tests via compose against deployable runtime image"} test-parallel = {shell = "poe test-backend & poe test-frontend & wait", help = "Run unit tests in parallel for speed"} # === Comprehensive Testing === test-unit = ["test-backend", "test-frontend"] diff --git a/backend/tests/integration/test_api.py b/backend/tests/integration/test_api.py index 176ea23..3f22b99 100644 --- a/backend/tests/integration/test_api.py +++ b/backend/tests/integration/test_api.py @@ -14,7 +14,6 @@ from backend.main import app, compatibility_status, get_api_session client = TestClient(app) -@pytest.mark.integration class TestAPIIntegration: """Integration tests for API endpoints.""" diff --git a/backend/tests/integration/test_runtime_blackbox.py b/backend/tests/integration/test_runtime_blackbox.py new file mode 100644 index 0000000..7919961 --- /dev/null +++ b/backend/tests/integration/test_runtime_blackbox.py @@ -0,0 +1,53 @@ +"""Runtime black-box integration tests executed over the container network.""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request + +import pytest + +BACKEND_BASE_URL = os.getenv("INTEGRATION_BACKEND_URL", "http://backend:8000").rstrip( + "/" +) +pytestmark = pytest.mark.integration + + +def _fetch(path: str) -> tuple[int, str]: + """Fetch an endpoint and return status code and body text.""" + url = f"{BACKEND_BASE_URL}{path}" + request = urllib.request.Request(url=url, method="GET") + try: + with urllib.request.urlopen(request, timeout=5) as response: + body = response.read().decode("utf-8", errors="replace") + return response.status, body + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + return exc.code, body + + +def test_root_endpoint_runtime() -> None: + """Runtime root endpoint should return backend API message.""" + status, body = _fetch("/") + assert status == 200 + payload = json.loads(body) + assert payload == {"message": "Plex Playlist Backend API"} + + +def test_health_endpoint_runtime() -> None: + """Runtime health endpoint should report healthy database connectivity.""" + status, body = _fetch("/health") + assert status == 200 + payload = json.loads(body) + assert payload.get("status") == "healthy" + assert payload.get("database") == "connected" + + +def test_compatibility_endpoint_runtime() -> None: + """Runtime compatibility endpoint should report policy check success.""" + status, body = _fetch("/compatibility") + assert status == 200 + payload = json.loads(body) + assert payload.get("ok") is True diff --git a/cicd-workflow.ods b/cicd-workflow.ods new file mode 100644 index 0000000..ff06fef Binary files /dev/null and b/cicd-workflow.ods differ diff --git a/compose.ci.e2e.yml b/compose.ci.e2e.yml new file mode 100644 index 0000000..9e98761 --- /dev/null +++ b/compose.ci.e2e.yml @@ -0,0 +1,28 @@ +--- +services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD:-plex_password} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local} + environment: + DATABASE_URL: ${DB_URL:-postgresql://plex_user:plex_password@database:5432/plex_playlist} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + + frontend: + image: ${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local} + depends_on: + backend: + condition: service_started diff --git a/compose.ci.integration.yml b/compose.ci.integration.yml new file mode 100644 index 0000000..cbbfc8d --- /dev/null +++ b/compose.ci.integration.yml @@ -0,0 +1,22 @@ +--- +services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD:-plex_password} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local} + environment: + DATABASE_URL: ${DB_URL:-postgresql://plex_user:plex_password@database:5432/plex_playlist} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy diff --git a/docs/CICD_MULTI_STAGE_BUILD.md b/docs/CICD_MULTI_STAGE_BUILD.md index 9f1691d..6092969 100644 --- a/docs/CICD_MULTI_STAGE_BUILD.md +++ b/docs/CICD_MULTI_STAGE_BUILD.md @@ -142,14 +142,10 @@ jobs: ### Responsibility Split -- `.gitea/workflows/cicd-start.yaml` owns startup routing and trace propagation. -- `.gitea/workflows/cicd-source-checks.yaml` owns early source-level - format/lint/type gating before any promotion dispatch. -- `.gitea/workflows/docker-build-base.yaml` owns base publication and verification. -- `.gitea/workflows/docker-build-main.yaml` owns complete-image publication. -- `.gitea/workflows/cicd-checks.yaml` and `.gitea/workflows/cicd-tests.yaml` - own post-build CI validation and tests. -- Main CI never rebuilds the base image locally. +- `.gitea/workflows/cicd.yaml` owns the full CI pipeline: base image publish, + CICD image publish, source checks, unit tests, runtime image publication, + integration tests, and E2E tests. +- The older split workflows remain only as deprecated/manual helper paths. ### Runtime Boundary Enforcement @@ -173,7 +169,7 @@ and deployable runtime artifacts before publishing the complete CICD image. Workflow location: -- `.gitea/workflows/docker-build-main.yaml` +- `.gitea/workflows/cicd.yaml` - `Verify deployable runtime boundaries` - `Build and verify deployable runtime image purity` diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 0c33cd2..b9caea0 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -30,6 +30,7 @@ Excluded: ### Runtime Artifact Definition - Container build source: `Dockerfile.backend`. +- Build shape: two-stage build with a dependency stage and a minimal runtime stage. - Runtime base image: `python:3.14-slim`. - Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`. - Exposed runtime port: `8000`. @@ -79,6 +80,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth. ### Frontend Runtime Artifact Definition - Container build source: `Dockerfile.frontend` (target `production`). +- Build shape: two-stage build with a dependency/build stage and a minimal nginx runtime stage. - Runtime base image: `nginx:alpine`. - Runtime process: `nginx -g "daemon off;"`. - Exposed runtime port: `80`. @@ -151,14 +153,14 @@ Current enforcement implemented in CI: - Dockerfile target boundary checks: - Script: `scripts/check-dockerfile-boundaries.sh` - - Workflow: `.gitea/workflows/docker-build-main.yaml` + - Workflow: `.gitea/workflows/cicd.yaml` - Deployable runtime image purity checks: - Script: `scripts/verify-deployable-image-purity.sh` - - Workflow: `.gitea/workflows/docker-build-main.yaml` + - Workflow: `.gitea/workflows/cicd.yaml` - Checks include binary presence and profile-specific package metadata probes to detect CI/development tooling leakage. - Runtime black-box integration checks against deployed backend container: - - Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job) + - Workflow: `.gitea/workflows/cicd.yaml` (`integration-tests` job) - Inputs: deployable backend commit tag reference and immutable digest reference from main build dispatch. - Assertions: digest/tag consistency and live endpoint behavior for `/`, diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d528284..82a4f75 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -36,7 +36,7 @@ Scope boundary: - Contract definition lives in `DEPLOYABLE_RUNTIME_CONTRACT.md`. - CI enforcement now includes Dockerfile boundary checks and deployable image - purity checks in `.gitea/workflows/docker-build-main.yaml`. + purity checks in `.gitea/workflows/cicd.yaml`. - Broader workflow redesign and deployment wiring remain out of scope for this repository's runtime contract document and belong to follow-up work under epic #66. @@ -60,8 +60,8 @@ Automated checks: and fails if CI/development binaries or package metadata artifacts are present. -These checks run in `.gitea/workflows/docker-build-main.yaml` before publishing -the complete CICD image. +These checks run in `.gitea/workflows/cicd.yaml` before publishing the +complete CICD image. ## Quick Start diff --git a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md index 54803c3..4bec6d4 100644 --- a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md +++ b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md @@ -175,10 +175,20 @@ If you want a failover strategy, keep the cached image tagged in the local regis Automation scripts for this workflow live in `scripts/gitea-actions/`: - `scripts/gitea-actions/repair_runner_mirror.xsh` - Repairs the mirror by pulling from GHCR, tagging/pushing to local registry, and verifying tag presence on each runner host. + Repairs mirrors by pulling upstream images, tagging/pushing to local registry, and verifying pullability on each runner host. + Current mirror targets: + - `kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest` + - `kankali.darkhelm.lan:3001/darkhelm.org/renovate:41` + - `kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy` - `scripts/gitea-actions/check_runner_images.xsh` - Verifies host-by-host image presence for both upstream (`GHCR`) and mirrored (`MIRROR`) tags. + Verifies host-by-host image presence and pullability for `ubuntu:22.04`, upstream act image (`GHCR`), mirrored act image (`MIRROR`), and Renovate image. + +- `scripts/gitea-actions/runner-prechange-capture.xsh` + Captures restart counts, OOM flags, memory state, and runner logs across all hosts. + +- `scripts/gitea-actions/diagnose_runner_startup.xsh` + Focused startup diagnostics for runner containers and mirror pull behavior. Run from repository root (xonsh): diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 1e05aeb..4f165a4 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -1,5 +1,10 @@ import { defineConfig, devices } from '@playwright/test'; +const runtimeBaseURL = process.env.PLAYWRIGHT_BASE_URL; +const useRuntimeServices = Boolean(runtimeBaseURL); +const junitOutputFile = process.env.PLAYWRIGHT_JUNIT_OUTPUT_FILE ?? 'playwright-results.xml'; +const outputDir = process.env.PLAYWRIGHT_OUTPUT_DIR ?? 'playwright-artifacts'; + export default defineConfig({ testDir: './tests/e2e', timeout: process.env.CI ? 60 * 1000 : 30 * 1000, // Longer timeout in CI @@ -10,11 +15,10 @@ export default defineConfig({ forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, workers: process.env.CI ? 1 : undefined, - reporter: process.env.CI - ? [['list'], ['junit', { outputFile: 'playwright-results.xml' }]] - : 'html', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: junitOutputFile }]] : 'html', + outputDir, use: { - baseURL: 'http://localhost:5173', + baseURL: runtimeBaseURL ?? 'http://localhost:5173', trace: 'on-first-retry', headless: process.env.CI ? true : false, // CI-specific browser optimizations @@ -78,12 +82,16 @@ export default defineConfig({ }, }, ], - webServer: { - command: 'yarn dev --host 0.0.0.0 --port 5173 --strictPort', - url: 'http://localhost:5173', - reuseExistingServer: !process.env.CI, - timeout: process.env.CI ? 180 * 1000 : 120 * 1000, // Longer startup timeout in CI - stderr: 'pipe', - stdout: 'pipe', - }, + ...(useRuntimeServices + ? {} + : { + webServer: { + command: 'yarn dev --host 0.0.0.0 --port 5173 --strictPort', + url: 'http://localhost:5173', + reuseExistingServer: !process.env.CI, + timeout: process.env.CI ? 180 * 1000 : 120 * 1000, // Longer startup timeout in CI + stderr: 'pipe', + stdout: 'pipe', + }, + }), }); diff --git a/renovate.json b/renovate.json index 205c4d0..d89213e 100644 --- a/renovate.json +++ b/renovate.json @@ -11,7 +11,14 @@ "prConcurrentLimit": 5, "branchConcurrentLimit": 10, "prHourlyLimit": 2, - + "enabledManagers": [ + "docker-compose", + "dockerfile", + "github-actions", + "npm", + "pep621", + "regex" + ], "packageRules": [ { "description": "Automerge non-major updates for high-confidence packages", @@ -28,9 +35,24 @@ }, { "description": "Group Python dev tools updates", - "matchManagers": ["uv"], + "matchManagers": ["pep621"], "matchCategories": ["python"], - "matchDepTypes": ["dev-dependencies"], + "matchPackageNames": [ + "ruff", + "pyright", + "pydoclint", + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "pre-commit", + "pyyaml", + "yamllint", + "toml-sort", + "language-formatters-pre-commit-hooks", + "xdoctest", + "poethepoet" + ], "groupName": "Python dev tools", "schedule": ["before 9am on monday"] }, @@ -76,7 +98,6 @@ "reviewersFromCodeOwners": true } ], - "customManagers": [ { "description": "Update Python version in Dockerfiles", @@ -97,19 +118,15 @@ "versioningTemplate": "node" } ], - - "osvVulnerabilityAlerts": true, + "osvVulnerabilityAlerts": false, "vulnerabilityAlerts": { "enabled": true, "schedule": ["at any time"] }, - "labels": ["dependencies"], "commitMessagePrefix": "chore:", "commitMessageTopic": "{{depName}}", "commitMessageExtra": "to {{newVersion}}", "commitMessageSuffix": "", - - "prTitle": "{{commitMessagePrefix}} {{commitMessageAction}} {{commitMessageTopic}} {{commitMessageExtra}}", "prBodyTemplate": "This PR contains the following updates:\n\n| Package | Change | Age | Adoption | Passing | Confidence |\n|---|---|---|---|---|---|\n{{#each upgrades}}\n|{{depName}}|{{#if displayFrom}}`{{{displayFrom}}}` -> `{{{displayTo}}}`{{else}}`{{{newVersion}}}`{{/if}}|[![age](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/age-slim)](https://docs.renovatebot.com/merge-confidence/)|[![adoption](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/adoption-slim)](https://docs.renovatebot.com/merge-confidence/)|[![passing](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/compatibility-slim/{{currentVersion}})](https://docs.renovatebot.com/merge-confidence/)|[![confidence](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/confidence-slim/{{currentVersion}})](https://docs.renovatebot.com/merge-confidence/)|\n{{/each}}\n\n---\n\n### Release Notes\n\n{{#each upgrades}}\n{{#if hasReleaseNotes}}\n
\n{{depName}}\n\n{{#each releases}}\n#### {{title}}\n\n{{#if body}}\n{{body}}\n{{/if}}\n{{/each}}\n\n
\n{{/if}}\n{{/each}}\n\n### Configuration\n\n📅 **Schedule**: {{schedule}}\n\n🚦 **Automerge**: {{#if isAutomerge}}Enabled{{else}}Disabled{{/if}}\n\n♻ **Rebasing**: {{#if isRebasing}}Rebasing{{else}}Not rebasing{{/if}}\n\n👥 **Reviewers**: {{#if reviewers}}{{reviewers}}{{else}}None{{/if}}\n\n---\n\n*This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate).*" } diff --git a/scripts/gitea-actions/check_runner_images.xsh b/scripts/gitea-actions/check_runner_images.xsh index 5432232..c1d43e6 100644 --- a/scripts/gitea-actions/check_runner_images.xsh +++ b/scripts/gitea-actions/check_runner_images.xsh @@ -5,26 +5,63 @@ hosts = [ "pi-desktop.darkhelm.lan", ] +mirror_image = "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest" +ghcr_runner_image = "ghcr.io/catthehacker/ubuntu:act-latest" +renovate_image = "ghcr.io/renovatebot/renovate:41" +python_image = "python:3.14-slim" +node_image = "node:20-bookworm-slim" + # Check both upstream and mirrored tags on each runner host. remote_script = """ -r = !(docker image inspect ubuntu:22.04 > /dev/null 2> /dev/null) -print("UBUNTU22:present" if r.returncode == 0 else "UBUNTU22:missing") -r = !(docker image inspect ghcr.io/catthehacker/ubuntu:act-latest > /dev/null 2> /dev/null) -print("GHCR:present" if r.returncode == 0 else "GHCR:missing") -r = !(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null) -print("MIRROR_LOCAL:present" if r.returncode == 0 else "MIRROR_LOCAL:missing") +def local_state(label, image): + r = !(docker image inspect @(image) > /dev/null 2> /dev/null) + print(f"{label}:present" if r.returncode == 0 else f"{label}:missing") + +local_state("UBUNTU22", "ubuntu:22.04") +local_state("PYTHON", "__PYTHON_IMAGE__") +local_state("NODE", "__NODE_IMAGE__") +local_state("GHCR", "__GHCR_RUNNER_IMAGE__") +local_state("MIRROR_LOCAL", "__MIRROR_IMAGE__") +local_state("RENOVATE", "__RENOVATE_IMAGE__") r = !(docker info 2> /dev/null | grep -qi "kankali.darkhelm.lan:3001") print("REGISTRY_CONFIG:ok" if r.returncode == 0 else "REGISTRY_CONFIG:missing") -pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-pull.log 2>&1) -if pull.returncode == 0: - print("MIRROR_REMOTE:pull-ok") -else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-pull.log) - print("MIRROR_REMOTE:https-mismatch" if mismatch.returncode == 0 else "MIRROR_REMOTE:pull-failed") +def remote_pull(label, image, log_path): + pull = !(docker pull @(image) > @(log_path) 2>&1) + if pull.returncode == 0: + print(f"{label}:pull-ok") + return + mismatch = !(grep -qi "http response to https client" @(log_path)) + if mismatch.returncode == 0: + print(f"{label}:https-mismatch") + else: + print(f"{label}:pull-failed") + +remote_pull("UBUNTU22_REMOTE", "ubuntu:22.04", "/tmp/ubuntu22-pull.log") +remote_pull("PYTHON_REMOTE", "__PYTHON_IMAGE__", "/tmp/python-pull.log") +remote_pull("NODE_REMOTE", "__NODE_IMAGE__", "/tmp/node-pull.log") +remote_pull("GHCR_REMOTE", "__GHCR_RUNNER_IMAGE__", "/tmp/ghcr-runner-pull.log") +remote_pull("MIRROR_REMOTE", "__MIRROR_IMAGE__", "/tmp/mirror-pull.log") +remote_pull("RENOVATE_REMOTE", "__RENOVATE_IMAGE__", "/tmp/renovate-pull.log") + +local_state("UBUNTU22_AFTER_PULL", "ubuntu:22.04") +local_state("PYTHON_AFTER_PULL", "__PYTHON_IMAGE__") +local_state("NODE_AFTER_PULL", "__NODE_IMAGE__") +local_state("GHCR_AFTER_PULL", "__GHCR_RUNNER_IMAGE__") +local_state("MIRROR_AFTER_PULL", "__MIRROR_IMAGE__") +local_state("RENOVATE_AFTER_PULL", "__RENOVATE_IMAGE__") """ +remote_script = ( + remote_script + .replace("__MIRROR_IMAGE__", mirror_image) + .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image) + .replace("__RENOVATE_IMAGE__", renovate_image) + .replace("__PYTHON_IMAGE__", python_image) + .replace("__NODE_IMAGE__", node_image) +) + for host in hosts: print(f"\n=== {host} ===") ssh @(host) @(remote_script) diff --git a/scripts/gitea-actions/collect_runner_diagnostics.xsh b/scripts/gitea-actions/collect_runner_diagnostics.xsh index 3251a44..fc98e3d 100644 --- a/scripts/gitea-actions/collect_runner_diagnostics.xsh +++ b/scripts/gitea-actions/collect_runner_diagnostics.xsh @@ -4,7 +4,8 @@ # source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago" # source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234" -from datetime import datetime +from datetime import datetime, UTC +import subprocess hosts = [ "kankali.darkhelm.lan", @@ -16,53 +17,79 @@ hosts = [ since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago" trace = $ARGS[1] if len($ARGS) > 1 else "" -print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z") +print(f"runner-diagnostics-start={datetime.now(UTC).isoformat()}") print(f"since={since}") print(f"trace_filter={trace or 'none'}") -remote_script = f''' -set -eu -SINCE="{since}" -TRACE="{trace}" +remote_script = """ +import subprocess -echo "host=$(hostname -f 2>/dev/null || hostname)" -echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" -echo "kernel=$(uname -r 2>/dev/null || echo unknown)" +SINCE = __SINCE__ +TRACE = __TRACE__ -echo "=== system pressure ===" -free -h || true -df -h || true -uptime || true +def run(cmd: str): + proc = subprocess.run(cmd, shell=True, text=True, capture_output=True) + if proc.stdout: + print(proc.stdout, end="") + if proc.stderr: + print(proc.stderr, end="") + return proc.returncode -echo "=== docker runner containers ===" -docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match" +run('echo "host=$(hostname -f 2>/dev/null || hostname)"') +run('echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"') +run('echo "kernel=$(uname -r 2>/dev/null || echo unknown)"') -echo "=== runner container inspect ===" -for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do - echo "--- container=$c ---" - docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true - docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true -done +print('=== system pressure ===') +run('free -h || true') +run('df -h || true') +run('uptime || true') -echo "=== docker daemon recent log ===" -if command -v journalctl >/dev/null 2>&1; then - journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true -else - echo "journalctl_unavailable=true" -fi +print('=== docker runner containers ===') +ps_cmd = "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}' | grep -E 'gitea|runner|act'" +if run(ps_cmd) != 0: + print('no_runner_container_match') -echo "=== oom and kill signals ===" -dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true +print('=== runner container inspect ===') +names_cmd = "docker ps -a --format '{{.Names}}' | grep -E 'gitea|runner|act' || true" +names_proc = subprocess.run(names_cmd, shell=True, text=True, capture_output=True) +container_names = [line.strip() for line in (names_proc.stdout or '').splitlines() if line.strip()] -if [ -n "$TRACE" ]; then - echo "=== trace-filtered logs ===" - for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do - echo "--- trace search in container=$c ---" - docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true - done -fi -''' +for c in container_names: + print(f'--- container={c} ---') + run(f"docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' {c} || true") + run(f'docker logs --since "{SINCE}" --tail 200 {c} 2>&1 || true') + +print('=== docker daemon recent log ===') +if run('command -v journalctl >/dev/null 2>&1') == 0: + run(f'journalctl -u docker --since "{SINCE}" --no-pager -n 400 2>/dev/null || true') +else: + print('journalctl_unavailable=true') + +print('=== oom and kill signals ===') +run("dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true") + +if TRACE: + print('=== trace-filtered logs ===') + for c in container_names: + print(f'--- trace search in container={c} ---') + run(f'docker logs --since "{SINCE}" {c} 2>&1 | grep -F "{TRACE}" | tail -n 80 || true') +""" + +remote_script = remote_script.replace("__SINCE__", repr(since)).replace("__TRACE__", repr(trace)) for host in hosts: print(f"\n=== {host} ===") - ssh @(host) @(remote_script) + try: + result = subprocess.run( + ["ssh", host], + input=remote_script, + capture_output=True, + text=True, + check=False, + ) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="") + except Exception as exc: + print(f"runner_diagnostics_error host={host} error={exc}") diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 8186902..8cc067c 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -18,6 +18,15 @@ HOSTS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +WARMUP_IMAGES = [ + "ubuntu:22.04", + "python:3.14-slim", + "node:20-bookworm-slim", + "ghcr.io/catthehacker/ubuntu:act-latest", + "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", + "ghcr.io/renovatebot/renovate:41", +] + DEFAULT_MEM_FRACTION = 0.75 DEFAULT_JOB_MEM_FRACTION = 0.9 DEFAULT_HOST_MEM_SHARE = 0.25 @@ -258,6 +267,20 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr +def warm_runner_images(host): + pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) + rc, out = ssh( + host, + ( + "set -e\n" + "echo 'warming_runner_images_start'\n" + f"{pulls}\n" + "echo 'warming_runner_images_done'\n" + ), + ) + return rc, out + + def service_block_bounds(lines, start_index): service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" ")) block_start = start_index + 1 @@ -743,6 +766,13 @@ with tempfile.TemporaryDirectory() as tmpdir: continue print(out.strip()) + rc, out = warm_runner_images(host) + if rc != 0: + print(out.strip()) + print("Failed to warm runner images on host") + continue + print(out.strip()) + rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'") print("mounts:", out.strip()) diff --git a/scripts/gitea-actions/diagnose_runner_startup.xsh b/scripts/gitea-actions/diagnose_runner_startup.xsh index 8339c98..70ec106 100755 --- a/scripts/gitea-actions/diagnose_runner_startup.xsh +++ b/scripts/gitea-actions/diagnose_runner_startup.xsh @@ -10,6 +10,8 @@ hosts = { } mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest' +ghcr_runner_image = 'ghcr.io/catthehacker/ubuntu:act-latest' +renovate_image = 'ghcr.io/renovatebot/renovate:41' do_fix = '--fix' in sys.argv[1:] remote_diag = f""" @@ -19,6 +21,9 @@ print('[runner_containers]') r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') print(str(r.out).strip()) +r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}}') +runner_names = [line.strip() for line in str(r.out).splitlines() if line.strip()] + print('[registry_config]') r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001') print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing') @@ -36,31 +41,47 @@ else: tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log) print(str(tail_out.out).strip()) -print('[recent_logs_runner1]') -log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1) -log1_lines = [ - line for line in str(log1.out).splitlines() - if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) -] -print('\\n'.join(log1_lines[-60:])) +if not runner_names: + print('[recent_logs] no-runner-containers-found') -print('[recent_logs_runner2]') -log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1) -log2_lines = [ - line for line in str(log2.out).splitlines() - if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) -] -print('\\n'.join(log2_lines[-60:])) +for name in runner_names: + print(f'[recent_logs_{name}]') + log = !(docker logs --tail 60 @(name) 2>&1) + log_lines = [ + line for line in str(log.out).splitlines() + if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline', 'no route to host', 'network is unreachable']) + ] + print('\\n'.join(log_lines[-60:])) """ remote_fix = """ print('[fix] restarting runners') -$(docker compose up -d --force-recreate runner1 runner2) +r = !(docker compose up -d --force-recreate act_runner_1 act_runner_2 > /tmp/runner-restart.log 2>&1) +if r.returncode != 0: + r = !(docker compose up -d --force-recreate runner1 runner2 > /tmp/runner-restart.log 2>&1) + if r.returncode != 0: + tail_out = !(tail -n 60 /tmp/runner-restart.log) + print(str(tail_out.out).strip()) + raise SystemExit(1) $(sleep 3) +print('[fix] warming images') +$(docker pull ubuntu:22.04) +$(docker pull python:3.14-slim) +$(docker pull node:20-bookworm-slim) +$(docker pull __GHCR_RUNNER_IMAGE__) +$(docker pull __MIRROR_IMAGE__) +$(docker pull __RENOVATE_IMAGE__) r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') print(str(r.out).strip()) """ +remote_fix = ( + remote_fix + .replace("__MIRROR_IMAGE__", mirror_image) + .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image) + .replace("__RENOVATE_IMAGE__", renovate_image) +) + for host, path in hosts.items(): print(f"\n===== {host} =====") cmd = f"cd {path}\n{remote_diag}" diff --git a/scripts/gitea-actions/discover-runners.xsh b/scripts/gitea-actions/discover-runners.xsh index 137963c..5475f92 100755 --- a/scripts/gitea-actions/discover-runners.xsh +++ b/scripts/gitea-actions/discover-runners.xsh @@ -41,6 +41,17 @@ def first_existing_compose(host, root): return None +def list_runner_containers(host): + """Return actual runner container names present on a host.""" + rc, text = ssh( + host, + "docker ps -a --format '{{.Names}}' | grep '^gitea-act-runner-' || true", + ) + if rc != 0 or not text: + return [] + return [line.strip() for line in text.splitlines() if line.strip()] + + report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}" mkdir -p @(report_dir) report_path = os.path.join(report_dir, "discovery-report.txt") @@ -84,31 +95,29 @@ for host, root in HOSTS: _, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'") data["container_sizes"] = size_text - # Explicit runner names avoid label-filter parsing issues seen previously. - rc1, inspect_1 = ssh( - host, - "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'", - ) - rc2, inspect_2 = ssh( - host, - "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'", - ) - if rc1 != 0: - inspect_1 = "RUNNER1_NOT_FOUND" - if rc2 != 0: - inspect_2 = "RUNNER2_NOT_FOUND" - data["runner_inspect"] = (inspect_1 + "\n" + inspect_2).strip() + runner_containers = list_runner_containers(host) + data["runner_containers"] = runner_containers - log1_rc, log1 = ssh(host, "docker logs --tail=40 gitea-act-runner-1") - log2_rc, log2 = ssh(host, "docker logs --tail=40 gitea-act-runner-2") - if log1_rc != 0: - log1 = "RUNNER1_LOGS_UNAVAILABLE" - if log2_rc != 0: - log2 = "RUNNER2_LOGS_UNAVAILABLE" - data["runner_logs"] = { - "gitea-act-runner-1": log1, - "gitea-act-runner-2": log2, - } + inspect_lines = [] + runner_logs = {} + if not runner_containers: + inspect_lines.append("NO_RUNNER_CONTAINERS_FOUND") + for container in runner_containers: + rc, inspect_text = ssh( + host, + f"docker inspect {container} --format 'container={container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} image={{{{.Config.Image}}}}'", + ) + if rc != 0: + inspect_text = f"{container}_INSPECT_UNAVAILABLE" + inspect_lines.append(inspect_text) + + log_rc, log_text = ssh(host, f"docker logs --tail=40 {container}") + if log_rc != 0: + log_text = f"{container}_LOGS_UNAVAILABLE" + runner_logs[container] = log_text + + data["runner_inspect"] = "\n".join(inspect_lines).strip() + data["runner_logs"] = runner_logs results[host] = data print("Collected") @@ -134,8 +143,10 @@ with open(report_path, "w", encoding="utf-8") as handle: write_section(handle, "7. MEMORY STATUS", data.get("memory_status", "")) write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", "")) write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", "")) - write_section(handle, "10. RUNNER LOGS (1)", data.get("runner_logs", {}).get("gitea-act-runner-1", "")) - write_section(handle, "11. RUNNER LOGS (2)", data.get("runner_logs", {}).get("gitea-act-runner-2", "")) + for index, container in enumerate(data.get("runner_containers", []), start=10): + write_section(handle, f"{index}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, "")) + if not data.get("runner_containers"): + write_section(handle, "10. RUNNER LOGS", "NO_RUNNER_CONTAINERS_FOUND") with open(summary_path, "w", encoding="utf-8") as handle: handle.write("# Infrastructure Discovery Summary\n\n") @@ -144,12 +155,7 @@ with open(summary_path, "w", encoding="utf-8") as handle: handle.write("|------|------|---------|-----------------|\n") for host, data in results.items(): - visible = 0 - ps = data.get("docker_ps", "") - if "gitea-act-runner-1" in ps: - visible += 1 - if "gitea-act-runner-2" in ps: - visible += 1 + visible = len(data.get("runner_containers", [])) handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n") print("\nDiscovery complete") diff --git a/scripts/gitea-actions/repair_runner_mirror.xsh b/scripts/gitea-actions/repair_runner_mirror.xsh old mode 100644 new mode 100755 index 51dbbf7..6904ac0 --- a/scripts/gitea-actions/repair_runner_mirror.xsh +++ b/scripts/gitea-actions/repair_runner_mirror.xsh @@ -1,3 +1,5 @@ +#!/usr/bin/env xonsh + hosts = [ "kankali.darkhelm.lan", "zhokq.darkhelm.lan", @@ -5,38 +7,92 @@ hosts = [ "pi-desktop.darkhelm.lan", ] -# Pull upstream tag, retag/push to local registry mirror, verify mirror tag. -remote_code = """ -$(docker pull ubuntu:22.04) -$(docker pull ghcr.io/catthehacker/ubuntu:act-latest) -$(docker tag ghcr.io/catthehacker/ubuntu:act-latest kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest) +publisher_host = "kankali.darkhelm.lan" -push = !(docker push kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-push.log 2>&1) -if push.returncode == 0: - print("MIRROR_PUSH:ok") +# Pull upstream tags on the registry host, publish mirror tags there, then verify pullability on every host. +remote_code_template = """ +mirror_pairs = [ + ("ACT_UBUNTU", "ghcr.io/catthehacker/ubuntu:act-latest", "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest"), + ("RENOVATE", "ghcr.io/renovatebot/renovate:41", "kankali.darkhelm.lan:3001/darkhelm.org/renovate:41"), + ("PLAYWRIGHT", "mcr.microsoft.com/playwright:v1.56.1-jammy", "kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy"), +] + +is_publisher = __IS_PUBLISHER__ + +def classify_failure(prefix, log_path): + mismatch = !(grep -qi "http response to https client" @(log_path)) + if mismatch.returncode == 0: + print(f"{prefix}:https-mismatch") + else: + print(f"{prefix}:failed") + tail = !(tail -n 20 @(log_path) 2> /dev/null) + if tail.returncode == 0 and str(tail.out).strip(): + print(f"{prefix}_LOG_START") + print(str(tail.out).rstrip()) + print(f"{prefix}_LOG_END") + +def image_present(image): + result = !(docker image inspect @(image) > /dev/null 2>&1) + return result.returncode == 0 + +ubuntu_pull_log = "/tmp/ubuntu22-pull.log" +ubuntu_pull = !(docker pull ubuntu:22.04 > @(ubuntu_pull_log) 2>&1) +if ubuntu_pull.returncode == 0: + print("UBUNTU22_PULL:ok") else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-push.log) - print("MIRROR_PUSH:https-mismatch" if mismatch.returncode == 0 else "MIRROR_PUSH:failed") + classify_failure("UBUNTU22_PULL", ubuntu_pull_log) -# Validate that the mirror is actually pullable from this host's Docker configuration. -$(docker image rm kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null || true) -pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-verify.log 2>&1) -if pull.returncode == 0: - print("MIRROR_PULL:ok") -else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-verify.log) - print("MIRROR_PULL:https-mismatch" if mismatch.returncode == 0 else "MIRROR_PULL:failed") - -if $(docker image inspect ubuntu:22.04): +if image_present("ubuntu:22.04"): print("UBUNTU22_PRESENT") else: print("UBUNTU22_MISSING") -if $(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest): - print("MIRROR_PRESENT") -else: - print("MIRROR_MISSING") + +for label, source_image, mirror_image in mirror_pairs: + print(f"{label}_SOURCE={source_image}") + print(f"{label}_MIRROR={mirror_image}") + + src_pull_log = f"/tmp/{label.lower()}-source-pull.log" + mirror_push_log = f"/tmp/{label.lower()}-mirror-push.log" + mirror_verify_log = f"/tmp/{label.lower()}-mirror-verify.log" + + if is_publisher: + src_pull = !(docker pull @(source_image) > @(src_pull_log) 2>&1) + if src_pull.returncode == 0: + print(f"{label}_SOURCE_PULL:ok") + else: + classify_failure(f"{label}_SOURCE_PULL", src_pull_log) + continue + + tag = !(docker tag @(source_image) @(mirror_image) > /dev/null 2>&1) + if tag.returncode != 0: + print(f"{label}_TAG:failed") + continue + + push = !(docker push @(mirror_image) > @(mirror_push_log) 2>&1) + if push.returncode == 0: + print(f"{label}_MIRROR_PUSH:ok") + else: + classify_failure(f"{label}_MIRROR_PUSH", mirror_push_log) + continue + else: + print(f"{label}_SOURCE_PULL:skipped") + print(f"{label}_MIRROR_PUSH:skipped") + + # Validate mirror pullability with this host's Docker registry config. + rm_image = !(docker image rm @(mirror_image) > /dev/null 2>&1) + verify = !(docker pull @(mirror_image) > @(mirror_verify_log) 2>&1) + if verify.returncode == 0: + print(f"{label}_MIRROR_PULL:ok") + else: + classify_failure(f"{label}_MIRROR_PULL", mirror_verify_log) + + if image_present(mirror_image): + print(f"{label}_MIRROR_PRESENT") + else: + print(f"{label}_MIRROR_MISSING") """ for host in hosts: print(f"\n=== {host} ===") + remote_code = remote_code_template.replace("__IS_PUBLISHER__", "True" if host == publisher_host else "False") ssh @(host) @(remote_code) diff --git a/scripts/gitea-actions/rollout-one-runner.xsh b/scripts/gitea-actions/rollout-one-runner.xsh index afb5578..c97a8ef 100755 --- a/scripts/gitea-actions/rollout-one-runner.xsh +++ b/scripts/gitea-actions/rollout-one-runner.xsh @@ -13,6 +13,15 @@ HOST_PATHS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +WARMUP_IMAGES = [ + "ubuntu:22.04", + "python:3.14-slim", + "node:20-bookworm-slim", + "ghcr.io/catthehacker/ubuntu:act-latest", + "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", + "ghcr.io/renovatebot/renovate:41", +] + def usage(exit_code=1): print("Usage: xonsh rollout-one-runner.xsh [--batch]") @@ -99,5 +108,14 @@ for verify_cmd in [ rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'") print(out if rc == 0 else "runner2-removed") +print("Phase 5: warm images") +pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) +rc, out = ssh(f"set -e\n{pulls}") +if rc != 0: + print("Warmup failed") + print(out) + raise SystemExit(4) +print(out) + print("One-runner rollout complete for host") print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}") diff --git a/scripts/gitea-actions/runner-prechange-capture.xsh b/scripts/gitea-actions/runner-prechange-capture.xsh index 4609a54..527bd9b 100755 --- a/scripts/gitea-actions/runner-prechange-capture.xsh +++ b/scripts/gitea-actions/runner-prechange-capture.xsh @@ -37,6 +37,16 @@ def section(title, body): return f"\n{sep}\n{title}\n{sep}\n{body}\n" +def list_runner_containers(host): + rc, text = ssh( + host, + "docker ps -a --filter name=gitea-act-runner --format '{{.Names}}'", + ) + if rc != 0 or not text: + return [] + return [line.strip() for line in text.splitlines() if line.strip()] + + report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}" mkdir -p @(report_dir) report_path = os.path.join(report_dir, "capture.txt") @@ -57,39 +67,45 @@ for host, root in HOSTS.items(): rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'") data["ps"] = ps_text - rc1, inspect1 = ssh( - host, - "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", - ) - rc2, inspect2 = ssh( - host, - "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", - ) - if rc1 != 0: - inspect1 = "runner1-missing" - if rc2 != 0: - inspect2 = "runner2-missing" - data["inspect"] = inspect1 + "\n" + inspect2 + runner_containers = list_runner_containers(host) + data["runner_containers"] = runner_containers - rc, oom_scan = ssh( - host, - "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'", - ) - if rc != 0: - oom_scan = "oom-scan-unavailable" - data["oom_scan"] = oom_scan + inspect_lines = [] + oom_lines = [] + if not runner_containers: + inspect_lines.append("no-runner-containers-found") + oom_lines.append("no-runner-containers-found") + + for container in runner_containers: + rc, inspect_text = ssh( + host, + f"docker inspect {container} --format '{container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}'", + ) + if rc != 0: + inspect_text = f"{container}-inspect-unavailable" + inspect_lines.append(inspect_text) + + rc, oom_text = ssh( + host, + f"docker inspect {container} --format '{{{{.Name}}}} oom={{{{.State.OOMKilled}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}'", + ) + if rc != 0: + oom_text = f"{container}-oom-scan-unavailable" + oom_lines.append(oom_text) + + data["inspect"] = "\n".join(inspect_lines) + data["oom_scan"] = "\n".join(oom_lines) rc, mem = ssh(host, "free -h") data["mem"] = mem - rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1") - rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2") - if rc1 != 0: - log1 = "runner1-logs-unavailable" - if rc2 != 0: - log2 = "runner2-logs-unavailable" - data["log1"] = log1 - data["log2"] = log2 + runner_logs = {} + for container in runner_containers: + rc, log_text = ssh(host, f"docker logs --tail={TAIL_LINES} {container}") + if rc != 0: + log_text = f"{container}-logs-unavailable" + runner_logs[container] = log_text + data["runner_logs"] = runner_logs entries[host] = data @@ -108,8 +124,12 @@ with open(report_path, "w", encoding="utf-8") as handle: handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", ""))) handle.write(section("4. OOM SCAN", data.get("oom_scan", ""))) handle.write(section("5. MEMORY", data.get("mem", ""))) - handle.write(section("6. LOGS RUNNER1", data.get("log1", ""))) - handle.write(section("7. LOGS RUNNER2", data.get("log2", ""))) + containers = data.get("runner_containers", []) + if not containers: + handle.write(section("6. RUNNER LOGS", "no-runner-containers-found")) + else: + for idx, container in enumerate(containers, start=6): + handle.write(section(f"{idx}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, ""))) print("Capture complete") print(f"Report: {report_path}") diff --git a/scripts/run-ci-e2e-compose.sh b/scripts/run-ci-e2e-compose.sh new file mode 100644 index 0000000..bf0e55b --- /dev/null +++ b/scripts/run-ci-e2e-compose.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/compose.ci.e2e.yml" +PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-e2e-local}" + +mkdir -p "${ROOT_DIR}/frontend/.playwright-artifacts" + +export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}" +export DEPLOYABLE_FRONTEND_IMAGE="${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local}" +export DB_PASSWORD="${DB_PASSWORD:-plex_password}" +export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}" +export E2E_ARTIFACT_DIR="${E2E_ARTIFACT_DIR:-${ROOT_DIR}/frontend/.playwright-artifacts}" + +cleanup() { + docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +echo "Running E2E tests against runtime backend/frontend images via compose" +echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}" +echo "DEPLOYABLE_FRONTEND_IMAGE=${DEPLOYABLE_FRONTEND_IMAGE}" +echo "E2E_ARTIFACT_DIR=${E2E_ARTIFACT_DIR}" + +docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend + +docker run --rm \ + --network "${PROJECT_NAME}_default" \ + -e CI=true \ + -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ + -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ + -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ + -v "${ROOT_DIR}/frontend:/workspace/frontend" \ + -v "${E2E_ARTIFACT_DIR}:/tmp/playwright-artifacts" \ + mcr.microsoft/playwright:v1.56.1-jammy \ + bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" diff --git a/scripts/run-ci-integration-compose.sh b/scripts/run-ci-integration-compose.sh new file mode 100644 index 0000000..b5a3e46 --- /dev/null +++ b/scripts/run-ci-integration-compose.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/compose.ci.integration.yml" +PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-int-local}" + +export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}" +export DB_PASSWORD="${DB_PASSWORD:-plex_password}" +export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}" + +cleanup() { + docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +echo "Running runtime integration checks against deployable backend image via compose" +echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}" + +docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend + +fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")' + )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null + then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" +} + +backend_ready=false +for i in $(seq 1 40); do + backend_running_count="$(docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)" + if [ "${backend_running_count}" -eq 0 ]; then + echo "❌ Backend container exited before becoming healthy" + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 +done + +if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + exit 1 +fi + +root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" +if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + exit 1 +fi + +compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" +if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + exit 1 +fi + +echo "✅ Runtime integration checks passed"