diff --git a/.gitea/workflows/cicd-checks.yaml b/.gitea/workflows/cicd-checks.yaml index 8f9d076..6af6958 100644 --- a/.gitea/workflows/cicd-checks.yaml +++ b/.gitea/workflows/cicd-checks.yaml @@ -75,6 +75,33 @@ jobs: RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true + run-check: name: ${{ matrix.name }} # Run checks across the full act runner pool for maximum parallelism. @@ -288,6 +315,8 @@ jobs: echo "=== Pre-commit Hook End (success) ===" + - *failure_diagnostics_step + dispatch-tests: name: Dispatch CICD Tests # Prefer the generic ubuntu-latest label for better runner availability. @@ -386,3 +415,5 @@ jobs: done "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" + + - *failure_diagnostics_step diff --git a/.gitea/workflows/cicd-start.yaml b/.gitea/workflows/cicd-start.yaml index 11775c6..7c2a99f 100644 --- a/.gitea/workflows/cicd-start.yaml +++ b/.gitea/workflows/cicd-start.yaml @@ -201,3 +201,29 @@ jobs: done "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" + + - name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index deebf6f..f36eef9 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -75,6 +75,33 @@ jobs: RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true + backend-tests: name: Backend Tests # Use ubuntu-act runner pool for consistent availability. @@ -152,6 +179,8 @@ jobs: exit "${TEST_STATUS}" fi + - *failure_diagnostics_step + frontend-tests: name: Frontend Tests # Use ubuntu-act runner pool for consistent availability. @@ -193,6 +222,8 @@ jobs: exit "${TEST_STATUS}" fi + - *failure_diagnostics_step + xdoctest: name: Backend Doctests # Use ubuntu-act runner pool for consistent availability. @@ -235,6 +266,8 @@ jobs: exit "${TEST_STATUS}" fi + - *failure_diagnostics_step + integration-tests: name: Integration Tests # Use ubuntu-act runner pool for consistent availability. @@ -281,6 +314,8 @@ jobs: exit "${TEST_STATUS}" fi + - *failure_diagnostics_step + e2e-tests: name: End-to-End Tests # Use ubuntu-act runner pool for consistent availability. @@ -313,21 +348,26 @@ jobs: cd /workspace/frontend && if [ -d 'tests/e2e' ] || grep -q 'playwright' package.json; then yarn playwright --version && - browser_ok=false && - for i in 1 2 3; do - echo 'Playwright browser install attempt' \"\$i\"'/3' && - if timeout 1800 yarn playwright install chromium; then - browser_ok=true - break + PW_BROWSER_PATH="\${PLAYWRIGHT_BROWSERS_PATH:-/root/.cache/ms-playwright}" && + if find "\${PW_BROWSER_PATH}" -maxdepth 1 -type d -name 'chromium-*' | grep -q .; then + echo "Using preinstalled Playwright Chromium from \${PW_BROWSER_PATH}" + else + browser_ok=false && + for i in 1 2 3; do + echo 'Playwright browser install attempt' "\$i"'/3' && + if timeout 1800 yarn playwright install chromium; then + browser_ok=true + break + fi + if [ "\$i" -lt 3 ]; then + echo 'Playwright install attempt failed; retrying in 20s' + sleep 20 + fi + done && + if [ "\$browser_ok" != 'true' ]; then + echo '❌ Playwright browser install failed after 3 attempts' + exit 1 fi - if [ \"\$i\" -lt 3 ]; then - echo 'Playwright install attempt failed; retrying in 20s' - sleep 20 - fi - done && - if [ \"\$browser_ok\" != 'true' ]; then - echo '❌ Playwright browser install failed after 3 attempts' - exit 1 fi && yarn test:e2e --reporter=list --timeout=90000 else @@ -342,3 +382,5 @@ jobs: tail -n 200 "${LOG_FILE}" || true exit "${TEST_STATUS}" fi + + - *failure_diagnostics_step diff --git a/.gitea/workflows/cicd.yml b/.gitea/workflows/cicd.yml index d1893e4..f96a94b 100644 --- a/.gitea/workflows/cicd.yml +++ b/.gitea/workflows/cicd.yml @@ -113,3 +113,29 @@ jobs: done "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" + + - name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true diff --git a/.gitea/workflows/docker-build-base.yaml b/.gitea/workflows/docker-build-base.yaml index d5b8226..1aa7ba9 100644 --- a/.gitea/workflows/docker-build-base.yaml +++ b/.gitea/workflows/docker-build-base.yaml @@ -487,3 +487,29 @@ jobs: done "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" + + - name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml index a1f0309..d90e071 100644 --- a/.gitea/workflows/docker-build-main.yaml +++ b/.gitea/workflows/docker-build-main.yaml @@ -103,6 +103,33 @@ jobs: echo "curl unavailable; skipping reachability check" fi + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true + build: name: Build and Push CICD Complete Image # Use ubuntu-act runner pool for consistent availability. @@ -270,6 +297,8 @@ jobs: docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + - *failure_diagnostics_step + dispatch-checks: name: Dispatch CICD Checks runs-on: ubuntu-act @@ -368,3 +397,5 @@ jobs: done "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" + + - *failure_diagnostics_step diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index eebc574..7b99f93 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -265,6 +265,32 @@ jobs: echo "✓ Renovate execution completed" + - name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true + - name: Upload Renovate logs if: always() run: | diff --git a/.gitea/workflows/runner-canary.yaml b/.gitea/workflows/runner-canary.yaml index 7ccf86e..f3da5b7 100644 --- a/.gitea/workflows/runner-canary.yaml +++ b/.gitea/workflows/runner-canary.yaml @@ -76,6 +76,33 @@ jobs: echo "docker_preflight=skipped_for_${LABEL}" fi + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + else + echo "docker not available on this runner" + fi + + echo "=== Kernel Tail ===" + dmesg | tail -n 120 || true + canary-manual-heavy: name: Canary Heavy (${{ matrix.label }}) if: github.event_name == 'workflow_dispatch' @@ -141,3 +168,5 @@ jobs: tail -n 20 /tmp/canary-mirror-pull.log || true exit 1 fi + + - *failure_diagnostics_step diff --git a/Dockerfile.cicd b/Dockerfile.cicd index 42663d6..fd8de0e 100644 --- a/Dockerfile.cicd +++ b/Dockerfile.cicd @@ -209,7 +209,7 @@ RUN echo "=== Installing Pre-commit Hook Environments ===" && \ echo "No .pre-commit-config.yaml found, skipping hook installation"; \ fi -# PHASE 6: Playwright browsers optimization check (may be pre-installed in base image) +# PHASE 6: Playwright browsers optimization check WORKDIR /workspace/frontend RUN if [ -f ".frontend-deps-failed" ]; then \ echo "Frontend dependencies failed - Playwright E2E tests will be skipped"; \ @@ -218,11 +218,11 @@ RUN if [ -f ".frontend-deps-failed" ]; then \ # Check if Playwright CLI is available via yarn (from project dependencies) if yarn playwright --version >/dev/null 2>&1; then \ echo "✓ Playwright CLI available via yarn" && \ - # Check if browsers are pre-installed from base image - if yarn playwright install --dry-run >/dev/null 2>&1; then \ - echo "✓ Playwright browsers available from base image optimization"; \ + PW_BROWSER_PATH="${PLAYWRIGHT_BROWSERS_PATH:-/root/.cache/ms-playwright}" && \ + if find "${PW_BROWSER_PATH}" -maxdepth 1 -type d -name 'chromium-*' | grep -q .; then \ + echo "✓ Playwright Chromium browser preinstalled at ${PW_BROWSER_PATH}"; \ else \ - echo "⚠ Playwright browsers not pre-installed - will install on demand in CI"; \ + echo "⚠ Playwright Chromium not preinstalled at ${PW_BROWSER_PATH} - CI will install on demand"; \ fi; \ else \ echo "⚠ Playwright CLI not available - E2E setup will be handled in CI"; \ diff --git a/Dockerfile.cicd-base b/Dockerfile.cicd-base index 760fb5c..102f63d 100644 --- a/Dockerfile.cicd-base +++ b/Dockerfile.cicd-base @@ -131,10 +131,35 @@ RUN echo "=== Installing Global Development Tools ===" && \ which prettier && \ echo "✓ Global Node.js development tools installed via npm" -# Keep base image deterministic: browser binaries are installed at E2E runtime. -RUN echo "=== Skipping Playwright Browser Preinstall In Base Image ===" && \ - playwright --version && \ - echo "Browser binaries are installed in the E2E test job to avoid flaky base-image builds" +# Keep browser binaries in a deterministic location so E2E runs do not re-download them. +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +RUN echo "=== Pre-installing Playwright Chromium Browser ===" && \ + mkdir -p "${PLAYWRIGHT_BROWSERS_PATH}" && \ + browser_ok=false && \ + for i in 1 2 3; do \ + echo "Playwright browser install attempt ${i}/3" && \ + if timeout 1800 playwright install chromium; then \ + browser_ok=true; \ + break; \ + fi; \ + if [ "${i}" -lt 3 ]; then \ + echo "Playwright install attempt ${i} failed; retrying in 20s" && \ + sleep 20; \ + fi; \ + done && \ + if [ "${browser_ok}" != "true" ]; then \ + echo "❌ Failed to preinstall Playwright Chromium browser"; \ + exit 1; \ + fi && \ + chromium_count=$(find "${PLAYWRIGHT_BROWSERS_PATH}" -maxdepth 1 -type d -name 'chromium-*' | wc -l) && \ + echo "playwright_chromium_dir_count=${chromium_count} path=${PLAYWRIGHT_BROWSERS_PATH}" && \ + if [ "${chromium_count}" -eq 0 ]; then \ + echo "❌ No Chromium browser directories found after install"; \ + exit 1; \ + fi && \ + ls -la "${PLAYWRIGHT_BROWSERS_PATH}" && \ + echo "✓ Playwright Chromium preinstalled at ${PLAYWRIGHT_BROWSERS_PATH}" # Pre-install common Python development dependencies globally # These are stable tools that rarely change and take time to compile