From 7c06fa3474aff89e0f2492ccbf16b78ae054efd2 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 10:29:31 -0400 Subject: [PATCH 01/80] Refactor CI and runtime images to two-stage builds --- .dockerignore | 1 + .gitea/workflows/cicd-source-checks.yaml | 478 ------- .gitea/workflows/cicd-start.yaml | 228 ---- .gitea/workflows/cicd-tests.yaml | 575 -------- .gitea/workflows/cicd.yaml | 1172 +++++++++++++++++ .gitea/workflows/docker-build-base.yaml | 537 -------- .gitea/workflows/docker-build-main.yaml | 481 ------- Dockerfile.backend | 27 +- Dockerfile.frontend | 36 +- .../integration/test_runtime_blackbox.py | 55 + docs/CICD_MULTI_STAGE_BUILD.md | 14 +- docs/DEPLOYABLE_RUNTIME_CONTRACT.md | 8 +- docs/DEVELOPMENT.md | 6 +- frontend/playwright.config.ts | 32 +- 14 files changed, 1279 insertions(+), 2371 deletions(-) delete mode 100644 .gitea/workflows/cicd-source-checks.yaml delete mode 100644 .gitea/workflows/cicd-start.yaml delete mode 100644 .gitea/workflows/cicd-tests.yaml create mode 100644 .gitea/workflows/cicd.yaml delete mode 100644 .gitea/workflows/docker-build-base.yaml delete mode 100644 .gitea/workflows/docker-build-main.yaml create mode 100644 backend/tests/integration/test_runtime_blackbox.py diff --git a/.dockerignore b/.dockerignore index c6e9b4e..1dd3662 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,6 +25,7 @@ secrets/ .ruff_cache/ **/__pycache__/ **/.pytest_cache/ +**/.venv/ **/node_modules/ **/dist/ **/build/ diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml deleted file mode 100644 index 0b2fe3e..0000000 --- a/.gitea/workflows/cicd-source-checks.yaml +++ /dev/null @@ -1,478 +0,0 @@ -name: CICD Source Checks - -on: - workflow_dispatch: - inputs: - head_sha: - description: Commit SHA to process - required: false - source_workflow: - description: Upstream workflow name - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - base_needed: - description: Whether base rebuild is required downstream - required: false - base_hash: - description: Immutable base hash to pass downstream - required: false - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -concurrency: - group: source-checks-${{ github.sha }} - cancel-in-progress: true - -jobs: - setup: - name: Setup Source Checks Context - runs-on: ubuntu-act - timeout-minutes: 10 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - trace_id: ${{ steps.meta.outputs.trace_id }} - base_needed: ${{ steps.meta.outputs.base_needed }} - base_hash: ${{ steps.meta.outputs.base_hash }} - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - BASE_NEEDED_INPUT: ${{ github.event.inputs.base_needed }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - RESOLVED_BASE_NEEDED="${BASE_NEEDED_INPUT:-true}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-source-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Source Checks ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "base_needed=${RESOLVED_BASE_NEEDED}" - echo "base_hash=${BASE_HASH_INPUT:-deferred}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve source check metadata - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - BASE_NEEDED_INPUT: ${{ github.event.inputs.base_needed }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - RESOLVED_TRACE_ID="${TRACE_ID_INPUT:-cicd-source-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - RESOLVED_BASE_NEEDED="${BASE_NEEDED_INPUT:-true}" - RESOLVED_BASE_HASH="${BASE_HASH_INPUT:-deferred}" - - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - echo "trace_id=${RESOLVED_TRACE_ID}" >> "$GITHUB_OUTPUT" - echo "base_needed=${RESOLVED_BASE_NEEDED}" >> "$GITHUB_OUTPUT" - echo "base_hash=${RESOLVED_BASE_HASH}" >> "$GITHUB_OUTPUT" - - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true - - source-precommit-checks-backend: - name: Source Pre-commit Checks (backend) - runs-on: ubuntu-act - timeout-minutes: 40 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Checkout source snapshot - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -e - umask 022 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- . - echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" - else - git checkout HEAD -- . - echo "Falling back to default branch HEAD for source checks checkout" - fi - - test -f .pre-commit-config.yaml || { - echo "❌ Missing .pre-commit-config.yaml after checkout" - exit 1 - } - - test -f backend/pyproject.toml || { - echo "❌ Missing backend/pyproject.toml after checkout" - exit 1 - } - - test -f frontend/package.json || { - echo "❌ Missing frontend/package.json after checkout" - find . -maxdepth 3 -type f | sort | head -n 80 - exit 1 - } - - - name: Bootstrap backend toolchain - run: | - set -euo pipefail - - if ! command -v curl >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if ! command -v uv >/dev/null 2>&1; then - curl -LsSf https://astral.sh/uv/install.sh | sh - fi - export PATH="$HOME/.local/bin:$PATH" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Install backend dependencies - run: | - set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - export UV_LINK_MODE=copy - - cd backend - uv sync --dev - - - name: Run backend pre-commit source checks - env: - CI: "true" - SKIP: "eslint,prettier,typescript-check,tsdoc-lint" - run: | - set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - cd backend - uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml - - - *failure_diagnostics_step - - source-precommit-checks-frontend: - name: Source Pre-commit Checks (frontend) - runs-on: ubuntu-act - timeout-minutes: 40 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Checkout source snapshot - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -e - umask 022 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- . - echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" - else - git checkout HEAD -- . - echo "Falling back to default branch HEAD for source checks checkout" - fi - - test -f .pre-commit-config.yaml || { - echo "❌ Missing .pre-commit-config.yaml after checkout" - exit 1 - } - - test -f frontend/package.json || { - echo "❌ Missing frontend/package.json after checkout" - find . -maxdepth 3 -type f | sort | head -n 80 - exit 1 - } - - - name: Bootstrap frontend toolchain - run: | - set -euo pipefail - - if ! command -v curl >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if ! command -v uv >/dev/null 2>&1; then - curl -LsSf https://astral.sh/uv/install.sh | sh - fi - export PATH="$HOME/.local/bin:$PATH" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - if ! command -v node >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq nodejs npm - fi - if ! command -v corepack >/dev/null 2>&1; then - npm install -g corepack - fi - - corepack enable - - - name: Install frontend dependencies - run: | - set -euo pipefail - export NODE_OPTIONS="--max-old-space-size=512" - export YARN_NETWORK_CONCURRENCY=1 - cd frontend - if ! yarn install --immutable --mode=skip-build; then - echo "Yarn install failed; retrying with npm fallback for constrained runner memory" - rm -f package-lock.json - npm install --ignore-scripts --no-audit --no-fund --prefer-offline - fi - - - name: Run frontend pre-commit source checks - env: - CI: "true" - run: | - set -euo pipefail - cd frontend - corepack yarn eslint . --max-warnings=0 - corepack yarn vue-tsc --noEmit - corepack yarn prettier --check . - - - *failure_diagnostics_step - - dispatch-build: - name: Dispatch Downstream Build - runs-on: ubuntu-act - timeout-minutes: 10 - needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend] - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Dispatch downstream workflow - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - BASE_NEEDED: ${{ needs.setup.outputs.base_needed }} - BASE_HASH: ${{ needs.setup.outputs.base_hash }} - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - TRACE_ID: ${{ needs.setup.outputs.trace_id }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - - if [ "${BASE_NEEDED}" = "true" ]; then - TARGET_WORKFLOW="docker-build-base.yaml" - else - TARGET_WORKFLOW="docker-build-main.yaml" - fi - - echo "route_decision workflow=${TARGET_WORKFLOW} head_sha=${HEAD_SHA} base_needed=${BASE_NEEDED} trace_id=${TRACE_ID}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - if [ -n "${GITEA_SSH_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_IP:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - fi - - ensure_curl() { - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - echo "❌ curl is required for dispatch and could not be installed" - return 1 - } - - ensure_curl - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-source-checks" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "${TARGET_WORKFLOW}" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Source Checks" - --trace-id "${TRACE_ID}" - --base-needed "${BASE_NEEDED}" - --base-hash "${BASE_HASH}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - echo "✅ Source checks passed; dispatching ${TARGET_WORKFLOW}" - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - *failure_diagnostics_step diff --git a/.gitea/workflows/cicd-start.yaml b/.gitea/workflows/cicd-start.yaml deleted file mode 100644 index 1fa74e5..0000000 --- a/.gitea/workflows/cicd-start.yaml +++ /dev/null @@ -1,228 +0,0 @@ -name: CICD Start - -on: - push: - branches: [ '**' ] - pull_request: - branches: [ main, develop ] - workflow_dispatch: - inputs: - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -concurrency: - group: start-${{ github.ref }} - cancel-in-progress: true - -jobs: - sanity-and-routing: - name: Sanity and Base Decision - # Keep startup lightweight to avoid runner image setup stalls. - # Use the same stable self-hosted label as the downstream CICD jobs. - runs-on: ubuntu-act - timeout-minutes: 12 - outputs: - base_needed: ${{ steps.base-decision.outputs.base_needed }} - head_sha: ${{ steps.base-decision.outputs.head_sha }} - base_hash: ${{ steps.base-decision.outputs.base_hash }} - - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - REPOSITORY: ${{ github.repository }} - ACTOR: ${{ github.actor }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - SHA: ${{ github.sha }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - TRACE_ID="${TRACE_ID_INPUT:-cicd-start-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${SHA:0:8}}" - echo "=== Dispatch Audit: CICD Start ===" - echo "event_name=${EVENT_NAME}" - echo "repository=${REPOSITORY}" - echo "actor=${ACTOR}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "sha=${SHA}" - echo "trace_id=${TRACE_ID}" - - - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - name: Compute routing signal - id: base-decision - run: | - set -e - - # Keep this workflow container-agnostic: do not rely on checkout/actions/node. - # docker-build-base handles canonical base hash/existence checks. - echo "head_sha=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" - echo "base_hash=deferred" >> "${GITHUB_OUTPUT}" - echo "base_needed=true" >> "${GITHUB_OUTPUT}" - - echo "Routing via docker-build-base.yaml" - echo "Base hash delegated to Docker Build Base workflow" - - - name: Dispatch downstream workflow - if: steps.base-decision.outcome == 'success' && steps.base-decision.outputs.head_sha != '' && steps.base-decision.outputs.base_needed != '' - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - BASE_NEEDED: ${{ steps.base-decision.outputs.base_needed }} - HEAD_SHA: ${{ steps.base-decision.outputs.head_sha }} - BASE_HASH: ${{ steps.base-decision.outputs.base_hash }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-start-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}" - - echo "trace_id=${TRACE_ID}" - echo "target_ref=${TARGET_REF}" - - TARGET_WORKFLOW="cicd-source-checks.yaml" - - echo "route_decision workflow=${TARGET_WORKFLOW} head_sha=${HEAD_SHA} base_needed=${BASE_NEEDED} trace_id=${TRACE_ID}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - if [ -n "${GITEA_SSH_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_HOST:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - fi - if [ -n "${GITEA_REGISTRY_IP:-}" ]; then - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - fi - - ensure_curl() { - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - if command -v curl >/dev/null 2>&1; then - return 0 - fi - - echo "❌ curl is required for dispatch and could not be installed" - return 1 - } - - ensure_curl - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-start" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "${TARGET_WORKFLOW}" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Start" - --trace-id "${TRACE_ID}" - --base-needed "${BASE_NEEDED}" - --base-hash "${BASE_HASH}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml deleted file mode 100644 index d29c6ad..0000000 --- a/.gitea/workflows/cicd-tests.yaml +++ /dev/null @@ -1,575 +0,0 @@ -name: CICD Tests - -on: - workflow_dispatch: - inputs: - head_sha: - description: Commit SHA to process - required: false - deployable_backend_tag_ref: - description: Deployable backend image tag reference - required: false - deployable_backend_digest_ref: - description: Deployable backend image digest reference - required: false - source_workflow: - description: Upstream workflow name - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -env: - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -concurrency: - group: tests-${{ github.sha }} - cancel-in-progress: true - -jobs: - setup: - name: Setup Tests Context - # Use the same stable runner label as the test jobs. - runs-on: ubuntu-act - timeout-minutes: 8 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }} - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Tests ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" - echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve head SHA - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT" - - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true - - backend-tests: - name: Backend Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 25 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - &configure_registry_host_step - name: Configure registry host resolution - run: | - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - - &ensure_cicd_image_step - name: Ensure CICD image is available - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" - if docker image inspect "${IMAGE}" >/dev/null 2>&1; then - echo "Using cached CICD image: ${IMAGE}" - else - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - pulled=false - for i in 1 2 3; do - echo "Pull attempt ${i}/3 for ${IMAGE}" - if docker pull "${IMAGE}"; then - pulled=true - break - fi - if [ "${i}" -lt 3 ]; then - sleep_seconds=$((5 * i)) - echo "Pull failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - done - - if [ "${pulled}" != "true" ]; then - echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}" - exit 1 - fi - fi - - - name: Run backend tests with coverage - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95 - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Backend tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of backend test output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - frontend-tests: - name: Frontend Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 25 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run frontend tests with coverage - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/frontend && - yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85 - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Frontend tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of frontend test output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - xdoctest: - name: Backend Doctests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 15 - needs: setup - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run backend doctests - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - uv run xdoctest src/ --quiet - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Backend doctests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of doctest output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - integration-tests: - name: Runtime Black-Box Integration Tests - # Pin integration tests to high-memory worker to reduce setup-stage runner churn. - runs-on: ubuntu-act-8gb - timeout-minutes: 20 - needs: [setup, backend-tests] - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - name: Run runtime black-box integration checks - env: - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }} - run: | - set -euo pipefail - set -o pipefail - - RUN_ID="${GITHUB_RUN_ID:-local}" - RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" - LOG_FILE="$(mktemp)" - NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}" - DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}" - BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}" - DB_PASSWORD="plex_password" - DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" - - fetch_backend_endpoint() { - endpoint_path="$1" - output_file="$2" - probe_tmp="$(mktemp)" - - if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \ - 'import sys' \ - 'import urllib.error' \ - 'import urllib.request' \ - 'endpoint_path = sys.argv[1]' \ - 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ - 'try:' \ - ' with urllib.request.urlopen(url, timeout=2) as response:' \ - ' body = response.read().decode("utf-8", errors="replace")' \ - ' print(response.status)' \ - ' print(body)' \ - 'except urllib.error.HTTPError as err:' \ - ' body = err.read().decode("utf-8", errors="replace")' \ - ' print(err.code)' \ - ' print(body)' \ - 'except Exception:' \ - ' print("000")' \ - ' print("")' - )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null - then - : >"${output_file}" - rm -f "${probe_tmp}" - echo "000" - return 0 - fi - - http_code="$(head -n 1 "${probe_tmp}")" - tail -n +2 "${probe_tmp}" >"${output_file}" - rm -f "${probe_tmp}" - echo "${http_code}" - } - - dump_failure_context() { - echo "=== Runtime Integration Failure Context ===" - docker ps -a || true - echo "--- Backend logs (tail 200) ---" - docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true - echo "--- Database logs (tail 200) ---" - docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true - echo "--- Backend inspect (status/image) ---" - docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true - echo "--- Database inspect (status/image) ---" - docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true - } - - cleanup() { - docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true - docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true - docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true - } - - trap cleanup EXIT - - { - if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Missing deployable backend image references from dispatch inputs" - exit 1 - fi - - echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" - - DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" - EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" - ACTUAL_TAG_DIGEST_REF="$({ - docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ - | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ - | head -n 1 - } || true)" - - if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then - echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" - exit 1 - fi - - if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then - echo "❌ Tag and digest mismatch" - echo "expected=${EXPECTED_DIGEST_REF}" - echo "actual=${ACTUAL_TAG_DIGEST_REF}" - exit 1 - fi - - echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" - echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" - - docker network create "${NETWORK_NAME}" - - docker run -d \ - --name "${DB_CONTAINER}" \ - --network "${NETWORK_NAME}" \ - -e POSTGRES_DB=plex_playlist \ - -e POSTGRES_USER=plex_user \ - -e POSTGRES_PASSWORD="${DB_PASSWORD}" \ - postgres:16-alpine - - db_ready=false - for i in $(seq 1 30); do - if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then - db_ready=true - break - fi - sleep 2 - done - if [ "${db_ready}" != "true" ]; then - echo "❌ Database did not become ready" - dump_failure_context - exit 1 - fi - - docker run -d \ - --name "${BACKEND_CONTAINER}" \ - --network "${NETWORK_NAME}" \ - -p 18000:8000 \ - -e DATABASE_URL="${DB_URL}" \ - -e ENVIRONMENT=production \ - "${EXPECTED_DIGEST_REF}" - - backend_ready=false - for i in $(seq 1 40); do - backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)" - if [ "${backend_running}" != "true" ]; then - echo "❌ Backend container exited before becoming healthy" - dump_failure_context - exit 1 - fi - - health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" - if [ "${health_code}" = "200" ]; then - backend_ready=true - break - fi - sleep 2 - done - if [ "${backend_ready}" != "true" ]; then - echo "❌ Backend did not become healthy" - cat /tmp/blackbox-health.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" - if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then - echo "❌ Root endpoint validation failed" - cat /tmp/blackbox-root.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" - if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then - echo "❌ Compatibility endpoint validation failed" - cat /tmp/blackbox-compatibility.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" - if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then - echo "❌ Health endpoint validation failed" - cat /tmp/blackbox-health.json 2>/dev/null || true - dump_failure_context - exit 1 - fi - - echo "✅ Runtime black-box integration checks passed" - } 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of runtime black-box integration output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step - - e2e-tests: - name: End-to-End Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act - timeout-minutes: 30 - needs: [setup, frontend-tests] - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run E2E tests - env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} - run: | - set -o pipefail - LOG_FILE="$(mktemp)" - - docker run --rm -e CI=true "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/frontend && - if [ -d 'tests/e2e' ] || grep -q 'playwright' package.json; then - yarn playwright --version && - PW_BROWSER_PATH="\${PLAYWRIGHT_BROWSERS_PATH:-/root/.cache/ms-playwright}" && - if find "\${PW_BROWSER_PATH}" -maxdepth 1 -type d -name 'chromium-*' | grep -q .; then - echo 'Using preinstalled Playwright Chromium from '"\${PW_BROWSER_PATH}" - else - browser_ok=false && - for i in 1 2 3; do - echo 'Playwright browser install attempt' "\$i"'/3' && - if timeout 1800 yarn playwright install chromium; then - browser_ok=true - break - fi - if [ "\$i" -lt 3 ]; then - echo 'Playwright install attempt failed; retrying in 20s' - sleep 20 - fi - done && - if [ "\$browser_ok" != 'true' ]; then - echo '❌ Playwright browser install failed after 3 attempts' - exit 1 - fi - fi && - yarn test:e2e --reporter=list --timeout=90000 - else - echo 'No E2E tests found' - fi - " 2>&1 | tee "${LOG_FILE}" - - TEST_STATUS=${PIPESTATUS[0]} - if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ E2E tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of E2E output ---" - tail -n 200 "${LOG_FILE}" || true - exit "${TEST_STATUS}" - fi - - - *failure_diagnostics_step diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml new file mode 100644 index 0000000..6d4a551 --- /dev/null +++ b/.gitea/workflows/cicd.yaml @@ -0,0 +1,1172 @@ +name: CICD + +on: + push: + branches: ["**"] + pull_request: + branches: [main, develop] + workflow_dispatch: + inputs: + head_sha: + description: Commit SHA to process + required: false + base_hash: + description: Immutable base hash to use for CICD base image + required: false + force_rebuild_base: + description: Force CICD base rebuild + required: false + default: "false" + trace_id: + description: Correlation id propagated across CICD jobs + required: false + +env: + GITEA_SSH_HOST: kankali.darkhelm.lan + GITEA_SSH_PORT: "2222" + GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git + GITEA_REGISTRY: kankali.darkhelm.lan:3001 + GITEA_REGISTRY_IP: 10.18.75.2 + GITEA_REGISTRY_HOST: kankali.darkhelm.lan + +defaults: + run: + shell: bash + +concurrency: + group: cicd-${{ github.ref }} + cancel-in-progress: true + +jobs: + publish-base: + name: Build and Publish CICD Base Image + runs-on: ubuntu-act + timeout-minutes: 35 + outputs: + base_hash: ${{ steps.base-state.outputs.base_hash }} + head_sha: ${{ steps.meta.outputs.head_sha }} + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Resolve head SHA + id: meta + env: + HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + HEAD_SHA_FALLBACK: ${{ github.sha }} + run: | + RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" + echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for base inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + else + git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + fi + + chmod +x scripts/compute-cicd-base-hash.sh + + - name: Compute base hash and inspect registry + id: base-state + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + FORCE_REBUILD: ${{ github.event.inputs.force_rebuild_base }} + run: | + set -euo pipefail + BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) + BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" + BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" + + echo "base_hash=${BASE_HASH}" >> "$GITHUB_OUTPUT" + echo "base_ref_hash=${BASE_REF_HASH}" >> "$GITHUB_OUTPUT" + echo "base_ref_latest=${BASE_REF_LATEST}" >> "$GITHUB_OUTPUT" + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + + if [ "${FORCE_REBUILD:-false}" = "true" ]; then + echo "needs_build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if timeout 900 docker pull "${BASE_REF_HASH}" >/dev/null 2>&1; then + echo "needs_build=false" >> "$GITHUB_OUTPUT" + else + echo "needs_build=true" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push base image + if: steps.base-state.outputs.needs_build == 'true' + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + BASE_HASH: ${{ steps.base-state.outputs.base_hash }} + BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }} + BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }} + run: | + set -euo pipefail + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" + docker pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" + + PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}") + + docker build --progress=plain -f Dockerfile.cicd-base \ + --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \ + --build-arg BASE_IMAGE_HASH="${BASE_HASH}" \ + --build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \ + -t cicd-base:latest . + + docker tag cicd-base:latest "${BASE_REF_HASH}" + docker tag cicd-base:latest "${BASE_REF_LATEST}" + docker push "${BASE_REF_HASH}" + docker push "${BASE_REF_LATEST}" + + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + fi + dmesg | tail -n 120 || true + + build_cicd: + name: Build and Push CICD Image + runs-on: ubuntu-act-8gb + needs: publish-base + timeout-minutes: 60 + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.publish-base.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for CICD build inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh + else + git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh + fi + chmod +x scripts/compute-cicd-base-hash.sh + + - name: Build and push complete CICD image + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} + BASE_HASH_FROM_BUILD: ${{ needs.publish-base.outputs.base_hash }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f /tmp/ssh_key' EXIT + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + BASE_HASH="${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}" + BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" + docker pull "${BASE_IMAGE}" + + echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key + chmod 600 /tmp/ssh_key + + docker build -f Dockerfile.cicd \ + --secret id=ssh_private_key,src=/tmp/ssh_key \ + --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \ + --build-arg GITHUB_SHA="${HEAD_SHA}" \ + --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ + -t cicd:latest . + + docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" + docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" + docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + + - *failure_diagnostics_step + + backend-tests: + name: Backend Tests + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act + timeout-minutes: 25 + needs: build_cicd + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - &configure_registry_host_step + name: Configure registry host resolution + run: | + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + - &ensure_cicd_image_step + name: Ensure CICD image is available + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + if docker image inspect "${IMAGE}" >/dev/null 2>&1; then + echo "Using cached CICD image: ${IMAGE}" + else + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + pulled=false + for i in 1 2 3; do + echo "Pull attempt ${i}/3 for ${IMAGE}" + if docker pull "${IMAGE}"; then + pulled=true + break + fi + if [ "${i}" -lt 3 ]; then + sleep_seconds=$((5 * i)) + echo "Pull failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + + if [ "${pulled}" != "true" ]; then + echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}" + exit 1 + fi + fi + + - name: Run backend tests with coverage + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace/backend && + source .venv/bin/activate && + uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95 + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Backend tests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of backend test output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + precommit-tests: + name: Pre-commit Checks + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act + timeout-minutes: 30 + needs: build_cicd + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run pre-commit checks + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace && + if [ ! -d .git ]; then + git init /workspace + fi && + /workspace/backend/.venv/bin/pre-commit run --all-files --show-diff-on-failure --config .pre-commit-config.yaml + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Pre-commit checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of pre-commit output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + frontend-tests: + name: Frontend Tests + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act + timeout-minutes: 25 + needs: build_cicd + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run frontend tests with coverage + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace/frontend && + yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85 + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Frontend tests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of frontend test output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + xdoctest: + name: Backend Doctests + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act + timeout-minutes: 15 + needs: build_cicd + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run backend doctests + env: + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + run: | + set -o pipefail + LOG_FILE="$(mktemp)" + + docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " + cd /workspace/backend && + source .venv/bin/activate && + uv run xdoctest src/ --quiet + " 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Backend doctests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of doctest output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + runtime_images: + name: Build and Publish Runtime Images + runs-on: ubuntu-act-8gb + needs: [build_cicd, precommit-tests, backend-tests, frontend-tests, xdoctest] + timeout-minutes: 45 + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} + deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} + deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for runtime image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- \ + .dockerignore \ + Dockerfile.backend \ + Dockerfile.frontend \ + backend \ + frontend \ + scripts/check-dockerfile-boundaries.sh \ + scripts/verify-deployable-image-purity.sh + else + git checkout HEAD -- \ + .dockerignore \ + Dockerfile.backend \ + Dockerfile.frontend \ + backend \ + frontend \ + scripts/check-dockerfile-boundaries.sh \ + scripts/verify-deployable-image-purity.sh + fi + + - name: Verify deployable runtime boundaries + run: | + set -euo pipefail + bash ./scripts/check-dockerfile-boundaries.sh + + - name: Build and verify deployable runtime image purity + env: + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" . + docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" . + + bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend + bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend + + - name: Push deployable backend runtime image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker push "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null + DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - name: Push deployable frontend runtime image + id: deployable_frontend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" + docker push "${DEPLOYABLE_FRONTEND_TAG_REF}" + docker pull "${DEPLOYABLE_FRONTEND_TAG_REF}" >/dev/null + DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" + echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + integration-tests: + name: Runtime Black-Box Integration Tests + # Pin integration tests to high-memory worker to reduce setup-stage runner churn. + runs-on: ubuntu-act-8gb + timeout-minutes: 20 + needs: runtime_images + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - *configure_registry_host_step + - name: Run runtime black-box integration checks via compose + env: + HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} + run: | + set -euo pipefail + set -o pipefail + + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" + LOG_FILE="$(mktemp)" + COMPOSE_FILE="/tmp/compose.ci.integration.yaml" + COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" + CICD_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + + if ! command -v docker >/dev/null 2>&1; then + echo "❌ Docker is required" + exit 1 + fi + if ! docker compose version >/dev/null 2>&1; then + echo "❌ docker compose plugin is required" + exit 1 + fi + + cat >"${COMPOSE_FILE}" <<'EOF' + services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE} + environment: + DATABASE_URL: ${DB_URL} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + + integration-tests: + image: ${CICD_IMAGE} + depends_on: + backend: + condition: service_started + environment: + CI: "true" + INTEGRATION_BACKEND_URL: http://backend:8000 + EOF + + fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")' + )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null + then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + + dump_failure_context() { + echo "=== Runtime Integration Failure Context ===" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true + echo "--- Compose logs (tail 200) ---" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true + } + + cleanup() { + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + { + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable backend image references from dispatch inputs" + exit 1 + fi + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + + docker pull "${CICD_IMAGE}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then + echo "❌ Tag and digest mismatch" + echo "expected=${EXPECTED_DIGEST_REF}" + echo "actual=${ACTUAL_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" + echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" + + export DB_PASSWORD DB_URL CICD_IMAGE DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}" + + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend + + db_ready=false + for i in $(seq 1 30); do + if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + backend_ready=false + for i in $(seq 1 40); do + backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)" + if [ "${backend_running_count}" -eq 0 ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" + if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then + echo "❌ Health endpoint validation failed" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm integration-tests \ + bash -c "cd /workspace/backend && source .venv/bin/activate && uv run pytest -v --tb=short tests/integration/test_runtime_blackbox.py" + + echo "✅ Runtime integration checks passed" + } 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ Runtime integration checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of runtime integration output ---" + tail -n 200 "${LOG_FILE}" || true + exit "${TEST_STATUS}" + fi + + - *failure_diagnostics_step + + e2e-tests: + name: End-to-End Tests + # Use ubuntu-act runner pool for consistent availability. + runs-on: ubuntu-act-8gb + timeout-minutes: 45 + needs: runtime_images + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - *configure_registry_host_step + - *ensure_cicd_image_step + - name: Run E2E tests against runtime services via compose + env: + HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} + DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_frontend_tag_ref }} + DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_frontend_digest_ref }} + run: | + set -euo pipefail + set -o pipefail + LOG_FILE="$(mktemp)" + ARTIFACT_DIR="$(mktemp -d)" + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" + COMPOSE_FILE="/tmp/compose.ci.e2e.yaml" + COMPOSE_PROJECT_NAME="plex-e2e-${RUN_ID}-${RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" + CICD_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + FRONTEND_URL_RUNNER="http://frontend:80" + + if ! command -v docker >/dev/null 2>&1; then + echo "❌ Docker is required" + exit 1 + fi + if ! docker compose version >/dev/null 2>&1; then + echo "❌ docker compose plugin is required" + exit 1 + fi + + cat >"${COMPOSE_FILE}" <<'EOF' + services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE} + environment: + DATABASE_URL: ${DB_URL} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + + frontend: + image: ${DEPLOYABLE_FRONTEND_IMAGE} + depends_on: + backend: + condition: service_started + + e2e-tests: + image: ${CICD_IMAGE} + depends_on: + frontend: + condition: service_started + environment: + CI: "true" + PLAYWRIGHT_BASE_URL: http://frontend:80 + PLAYWRIGHT_OUTPUT_DIR: /tmp/playwright-artifacts + PLAYWRIGHT_JUNIT_OUTPUT_FILE: /tmp/playwright-artifacts/playwright-results.xml + volumes: + - ${E2E_ARTIFACT_DIR}:/tmp/playwright-artifacts + EOF + + fetch_http() { + container_name="$1" + endpoint_path="$2" + output_file="$3" + probe_tmp="$(mktemp)" + + if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f\"http://127.0.0.1:8000{endpoint_path}\"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode(\"utf-8\", errors=\"replace\")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode(\"utf-8\", errors=\"replace\")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print(\"000\")' \ + ' print(\"\")')" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + + fetch_url_from_backend() { + target_url="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'target_url = sys.argv[1]' \ + 'try:' \ + ' with urllib.request.urlopen(target_url, timeout=2) as response:' \ + ' body = response.read().decode(\"utf-8\", errors=\"replace\")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode(\"utf-8\", errors=\"replace\")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print(\"000\")' \ + ' print(\"\")')" "${target_url}" >"${probe_tmp}" 2>/dev/null; then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + + dump_failure_context() { + echo "=== Runtime E2E Failure Context ===" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true + echo "--- Compose logs (tail 200) ---" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true + echo "--- Artifact directory ---" + find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true + } + + cleanup() { + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable runtime image references from dispatch inputs" + exit 1 + fi + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + docker pull "${CICD_IMAGE}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + docker pull "${DEPLOYABLE_FRONTEND_TAG_REF}" + docker pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_BACKEND_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_BACKEND_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_BACKEND_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve backend digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_BACKEND_TAG_DIGEST_REF}" != "${EXPECTED_BACKEND_DIGEST_REF}" ]; then + echo "❌ Backend tag and digest mismatch" + echo "expected=${EXPECTED_BACKEND_DIGEST_REF}" + echo "actual=${ACTUAL_BACKEND_TAG_DIGEST_REF}" + exit 1 + fi + + DEPLOYABLE_FRONTEND_REPO="${DEPLOYABLE_FRONTEND_DIGEST_REF%%@*}" + EXPECTED_FRONTEND_DIGEST_REF="${DEPLOYABLE_FRONTEND_DIGEST_REF}" + ACTUAL_FRONTEND_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve frontend digest from tag reference: ${DEPLOYABLE_FRONTEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" != "${EXPECTED_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Frontend tag and digest mismatch" + echo "expected=${EXPECTED_FRONTEND_DIGEST_REF}" + echo "actual=${ACTUAL_FRONTEND_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}" + echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}" + + export DB_PASSWORD DB_URL CICD_IMAGE E2E_ARTIFACT_DIR="${ARTIFACT_DIR}" + export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}" + export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}" + + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend + + db_ready=false + for i in $(seq 1 30); do + if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + backend_ready=false + for i in $(seq 1 40); do + backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)" + if [ "${backend_running_count}" -eq 0 ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + + health_code="$(fetch_http "backend" "/health" /tmp/e2e-backend-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/e2e-backend-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(fetch_http "backend" "/" /tmp/e2e-backend-root.json)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/e2e-backend-root.json; then + echo "❌ Backend root endpoint validation failed" + cat /tmp/e2e-backend-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + frontend_ready=false + for i in $(seq 1 30); do + frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)" + if [ "${frontend_running_count}" -gt 0 ]; then + frontend_root_code="$(fetch_url_from_backend "${FRONTEND_URL_RUNNER}" /tmp/e2e-frontend-root.html)" + if [ "${frontend_root_code}" = "200" ]; then + frontend_ready=true + break + fi + fi + sleep 2 + done + if [ "${frontend_ready}" != "true" ]; then + echo "❌ Frontend did not become reachable" + cat /tmp/e2e-frontend-root.html 2>/dev/null || true + dump_failure_context + exit 1 + fi + + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm e2e-tests \ + bash -c "cd /workspace/frontend && yarn test:e2e --reporter=list --timeout=90000" \ + 2>&1 | tee "${LOG_FILE}" + + TEST_STATUS=${PIPESTATUS[0]} + if [ "${TEST_STATUS}" -ne 0 ]; then + echo "❌ E2E tests failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of E2E output ---" + tail -n 200 "${LOG_FILE}" || true + echo "--- E2E artifact files ---" + find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true + dump_failure_context + exit "${TEST_STATUS}" + fi + + echo "=== E2E artifact files ===" + find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true + + - *failure_diagnostics_step diff --git a/.gitea/workflows/docker-build-base.yaml b/.gitea/workflows/docker-build-base.yaml deleted file mode 100644 index 35d1738..0000000 --- a/.gitea/workflows/docker-build-base.yaml +++ /dev/null @@ -1,537 +0,0 @@ -name: Docker Build Base - -on: - workflow_dispatch: - inputs: - force_rebuild: - description: Force a rebuild even when the immutable base tag already exists - required: false - default: "false" - head_sha: - description: Commit SHA to process - required: false - source_workflow: - description: Upstream workflow name - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -concurrency: - group: base-${{ github.sha }} - cancel-in-progress: true - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -defaults: - run: - shell: bash - -jobs: - publish-base: - name: Build and Publish CICD Base Image - # Use ubuntu-act runner pool which is known to be available. - runs-on: ubuntu-act - timeout-minutes: 35 - outputs: - base_hash: ${{ steps.base-state.outputs.base_hash }} - head_sha: ${{ steps.meta.outputs.head_sha }} - - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Emit startup diagnostics - shell: sh - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TARGET_LABEL: ubuntu-act - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_SUFFIX="$(printf '%s' "${RESOLVED_HEAD_SHA}" | cut -c1-8)" - TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TRACE_SUFFIX}}" - echo "=== Base Workflow Startup Audit ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "force_rebuild=${FORCE_REBUILD}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - echo "target_runner_label=${TARGET_LABEL}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "startup_audit=ok" - - - name: Preflight Docker and registry connectivity - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - run: | - set -e - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - echo "=== Docker Preflight ===" - command -v docker - docker --version - docker info >/tmp/docker-info.log 2>&1 || (echo "❌ docker info failed" && tail -n 40 /tmp/docker-info.log && exit 1) - grep -qi "${GITEA_REGISTRY}" /tmp/docker-info.log || echo "⚠ Registry not listed in docker info insecure registries: ${GITEA_REGISTRY}" - - echo "=== Registry Login Preflight ===" - if [ -z "${PACKAGE_ACCESS_TOKEN}" ]; then - echo "❌ PACKAGE_ACCESS_TOKEN is empty" - exit 1 - fi - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/tmp/docker-login.log 2>&1 || (echo "❌ Registry login preflight failed" && tail -n 40 /tmp/docker-login.log && exit 1) - echo "registry_login_preflight=ok" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Base Image ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "force_rebuild=${FORCE_REBUILD}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve head SHA - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - - - name: Minimal checkout for base inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - GITHUB_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - echo "=== Minimal Repository Checkout for Base Inputs ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if [ -n "${SSH_PRIVATE_KEY}" ]; then - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo "Loaded SSH key fingerprint: $(ssh-keygen -lf ~/.ssh/id_rsa | awk '{print $2}')" - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - else - echo "❌ SSH_PRIVATE_KEY is empty" - exit 1 - fi - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout \ - "${GITEA_REPO_SSH_URL}" . - - if [ -n "${GITHUB_SHA}" ] && \ - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh - echo "✓ Checked out base inputs from commit ${GITHUB_SHA}" - else - git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh - echo "⚠ Falling back to default branch HEAD for base inputs checkout" - fi - - chmod +x scripts/compute-cicd-base-hash.sh - rm -f ~/.ssh/id_rsa - - - name: Compute base hash and inspect registry - id: base-state - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} - run: | - echo "=== Computing CICD Base Image Hash ===" - - BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) - BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" - - echo "Base hash: ${BASE_HASH}" - echo "base_hash=${BASE_HASH}" >> $GITHUB_OUTPUT - echo "base_ref_hash=${BASE_REF_HASH}" >> $GITHUB_OUTPUT - echo "base_ref_latest=${BASE_REF_LATEST}" >> $GITHUB_OUTPUT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null 2>&1; then - echo "❌ Registry login failed during base inspection" - exit 1 - fi - - if [ "${FORCE_REBUILD}" = "true" ]; then - echo "Manual force rebuild requested" - echo "needs_build=true" >> $GITHUB_OUTPUT - exit 0 - fi - - for i in 1 2 3; do - echo "Existing base check ${i}/3 for ${BASE_REF_HASH}..." - # Manifest inspect is unreliable on some runners; use pull as truth. - if timeout 900 docker pull "${BASE_REF_HASH}" >/tmp/base-state-pull.log 2>&1; then - echo "✓ Immutable base image already exists and is pullable: ${BASE_REF_HASH}" - echo "needs_build=false" >> $GITHUB_OUTPUT - exit 0 - else - pull_exit=$? - echo "Existing base pull check failed with exit code ${pull_exit}" - if [ "${pull_exit}" -eq 124 ]; then - echo "Existing base pull timed out after 900s" - fi - if [ -s /tmp/base-state-pull.log ]; then - tail -n 20 /tmp/base-state-pull.log || true - fi - fi - - if [ "${i}" -lt 3 ]; then - sleep 10 - fi - done - - echo "ℹ Immutable base image missing or not yet pullable: ${BASE_REF_HASH}" - echo "needs_build=true" >> $GITHUB_OUTPUT - - - name: Write registry push helpers - run: | - cat > /tmp/registry-push-helpers.sh <<'EOF' - #!/bin/bash - - ensure_skopeo() { - if command -v skopeo >/dev/null 2>&1; then - return 0 - fi - - if command -v apt-get >/dev/null 2>&1; then - apt-get update && apt-get install -y skopeo - elif command -v apk >/dev/null 2>&1; then - apk add --no-cache skopeo - fi - - if ! command -v skopeo >/dev/null 2>&1; then - echo "❌ skopeo not available for fallback push" - return 1 - fi - } - - push_ref_with_fallback() { - ref="$1" - local_image="$2" - archive_path="$3" - registry_user="$4" - package_access_token="$5" - - for i in 1 2 3; do - echo "Docker push attempt ${i}/3 for ${ref}..." - if docker push "${ref}"; then - echo "✓ Docker push succeeded for ${ref}" - return 0 - fi - - if [ "${i}" -lt 3 ]; then - sleep 10 - fi - done - - echo "⚠ Docker push failed for ${ref}; trying skopeo fallback" - - if ! ensure_skopeo; then - return 1 - fi - - if [ ! -f "${archive_path}" ]; then - echo "Creating local image archive for fallback push..." - if ! docker save "${local_image}" -o "${archive_path}"; then - echo "❌ Failed to create docker archive for fallback push" - return 1 - fi - fi - - if skopeo copy \ - --dest-creds "${registry_user}:${package_access_token}" \ - --dest-tls-verify=false \ - "docker-archive:${archive_path}" \ - "docker://${ref}"; then - echo "✓ Skopeo fallback push succeeded for ${ref}" - return 0 - fi - - echo "❌ Skopeo fallback push failed for ${ref}" - return 1 - } - EOF - - chmod 700 /tmp/registry-push-helpers.sh - - - name: Build and push base image - if: steps.base-state.outputs.needs_build == 'true' - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - BASE_HASH: ${{ steps.base-state.outputs.base_hash }} - BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }} - BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }} - run: | - echo "=== Building CICD Base Image ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - echo "❌ Registry login failed before build" - exit 1 - fi - - PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" - echo "playwright_browsers_mirror_tag=${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" - - if ! docker pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" >/tmp/playwright-mirror-pull.log 2>&1; then - echo "❌ Required internal Playwright browsers mirror image is missing: ${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" - echo "Mirror this image into your internal registry before running base builds." - tail -n 80 /tmp/playwright-mirror-pull.log || true - exit 1 - fi - - PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 2>/dev/null || true) - if [ -z "${PLAYWRIGHT_BROWSERS_IMAGE}" ]; then - echo "❌ Failed to resolve immutable digest for ${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" - exit 1 - fi - - echo "playwright_browsers_image=${PLAYWRIGHT_BROWSERS_IMAGE}" - - export DOCKER_BUILDKIT=1 - - BUILD_TIMEOUT_SECONDS=5400 - BUILD_START_EPOCH=$(date +%s) - echo "docker_build_timeout_seconds=${BUILD_TIMEOUT_SECONDS}" - echo "docker_build_started_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - timeout "${BUILD_TIMEOUT_SECONDS}" docker build --progress=plain -f Dockerfile.cicd-base \ - --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \ - --build-arg BASE_IMAGE_HASH="${BASE_HASH}" \ - --build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \ - -t cicd-base:latest . - BUILD_EXIT_CODE=$? - if [ "${BUILD_EXIT_CODE}" -ne 0 ]; then - if [ "${BUILD_EXIT_CODE}" -eq 124 ]; then - echo "❌ docker build timed out after ${BUILD_TIMEOUT_SECONDS}s" - else - echo "❌ docker build failed with exit code ${BUILD_EXIT_CODE}" - fi - exit "${BUILD_EXIT_CODE}" - fi - - BUILD_END_EPOCH=$(date +%s) - BUILD_ELAPSED=$((BUILD_END_EPOCH - BUILD_START_EPOCH)) - echo "docker_build_elapsed_seconds=${BUILD_ELAPSED}" - - ARCHIVE_PATH="/tmp/cicd-base.tar" - source /tmp/registry-push-helpers.sh - - docker tag cicd-base:latest "${BASE_REF_HASH}" - docker tag cicd-base:latest "${BASE_REF_LATEST}" - - push_ref_with_fallback "${BASE_REF_HASH}" "cicd-base:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}" - push_ref_with_fallback "${BASE_REF_LATEST}" "cicd-base:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}" - - - name: Verify published base image - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }} - run: | - echo "=== Verifying Published CICD Base Image ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - echo "❌ Registry login failed during verification" - exit 1 - fi - - for i in 1 2 3 4 5 6; do - echo "Verification attempt ${i}/6 for ${BASE_REF_HASH}..." - - if docker pull "${BASE_REF_HASH}"; then - echo "✓ Published base image is pullable: ${BASE_REF_HASH}" - exit 0 - fi - - if [ "${i}" -lt 6 ]; then - echo "⚠ Pull failed; waiting 20s before retry" - sleep 20 - fi - done - - echo "❌ Published base image could not be pulled after 6 attempts: ${BASE_REF_HASH}" - exit 1 - - - name: Dispatch main build workflow - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH: ${{ steps.base-state.outputs.base_hash }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}" - - echo "trace_id=${TRACE_ID}" - echo "target_ref=${TARGET_REF}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - - if ! command -v curl >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-base" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "docker-build-main.yaml" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Base Image" - --trace-id "${TRACE_ID}" - --base-needed "true" - --base-hash "${BASE_HASH}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml deleted file mode 100644 index bdf21c2..0000000 --- a/.gitea/workflows/docker-build-main.yaml +++ /dev/null @@ -1,481 +0,0 @@ -name: Docker Build Main - -on: - workflow_dispatch: - inputs: - head_sha: - description: Commit SHA to process - required: false - base_hash: - description: Immutable base hash to use for CICD base image - required: false - source_workflow: - description: Upstream workflow name - required: false - base_needed: - description: Whether base rebuild was required - required: false - trace_id: - description: Correlation id propagated across CICD dispatch chain - required: false - -env: - GITEA_SSH_HOST: kankali.darkhelm.lan - GITEA_SSH_PORT: "2222" - GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git - GITEA_REGISTRY: kankali.darkhelm.lan:3001 - GITEA_REGISTRY_IP: 10.18.75.2 - GITEA_REGISTRY_HOST: kankali.darkhelm.lan - -defaults: - run: - shell: bash - -concurrency: - group: main-build-${{ github.sha }} - cancel-in-progress: true - -jobs: - startup-audit: - name: Main Workflow Startup Audit - # Pin startup audit to high-memory worker to avoid setup-stage runner churn. - runs-on: ubuntu-act-8gb - timeout-minutes: 5 - steps: - - name: Identify runner - shell: sh - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Emit startup diagnostics - shell: sh - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - BASE_NEEDED: ${{ github.event.inputs.base_needed }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - TARGET_LABEL: ubuntu-act-8gb - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_SUFFIX="$(printf '%s' "${RESOLVED_HEAD_SHA}" | cut -c1-8)" - TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TRACE_SUFFIX}}" - echo "=== Main Workflow Startup Audit ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "base_needed=${BASE_NEEDED}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "base_hash_input=${BASE_HASH_INPUT}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - echo "target_runner_label=${TARGET_LABEL}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "startup_audit=ok" - - - name: Check control-plane reachability - shell: sh - run: | - SERVER_URL="${GITHUB_SERVER_URL:-}" - if command -v curl >/dev/null 2>&1; then - if [ -n "${SERVER_URL}" ] && curl -fsS --connect-timeout 5 --max-time 10 "${SERVER_URL%/}/api/v1/version" >/tmp/gitea-version.json 2>/dev/null; then - echo "gitea_api_reachable=true" - cat /tmp/gitea-version.json || true - else - echo "gitea_api_reachable=false" - fi - else - echo "curl unavailable; skipping reachability check" - fi - - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - else - echo "docker not available on this runner" - fi - - echo "=== Kernel Tail ===" - dmesg | tail -n 120 || true - - build: - name: Build and Push CICD Complete Image - # Pin main image build to high-memory worker to reduce setup-time failures. - runs-on: ubuntu-act-8gb - needs: startup-audit - timeout-minutes: 60 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} - - steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - - name: Audit trigger context - env: - EVENT_NAME: ${{ github.event_name }} - SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} - BASE_NEEDED: ${{ github.event.inputs.base_needed }} - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - HEAD_REF: ${{ github.head_ref }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}" - echo "=== Dispatch Audit: CICD Main Build ===" - echo "event_name=${EVENT_NAME}" - echo "source_workflow=${SOURCE_WORKFLOW}" - echo "base_needed=${BASE_NEEDED}" - echo "head_sha_input=${HEAD_SHA_INPUT}" - echo "head_sha=${RESOLVED_HEAD_SHA}" - echo "base_hash_input=${BASE_HASH_INPUT}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "head_ref=${HEAD_REF}" - echo "trace_id=${TRACE_ID}" - - - name: Resolve head SHA - id: meta - env: - HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} - HEAD_SHA_FALLBACK: ${{ github.sha }} - run: | - RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" - echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - - - name: Minimal checkout for build and verification inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -e - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1 || true - - git checkout FETCH_HEAD -- \ - .dockerignore \ - Dockerfile.backend \ - Dockerfile.frontend \ - Dockerfile.cicd \ - Dockerfile.cicd-base \ - backend \ - frontend \ - scripts/compute-cicd-base-hash.sh \ - scripts/check-dockerfile-boundaries.sh \ - scripts/verify-deployable-image-purity.sh - chmod +x scripts/compute-cicd-base-hash.sh - - - name: Verify deployable runtime boundaries - run: | - set -e - bash ./scripts/check-dockerfile-boundaries.sh - - - name: Build and verify deployable runtime image purity - env: - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -e - - docker build -f Dockerfile.backend \ - -t deployable-backend:"${HEAD_SHA}" . - - docker build -f Dockerfile.frontend \ - --target production \ - -t deployable-frontend:"${HEAD_SHA}" . - - bash ./scripts/verify-deployable-image-purity.sh \ - --image deployable-backend:"${HEAD_SHA}" \ - --profile backend - - bash ./scripts/verify-deployable-image-purity.sh \ - --image deployable-frontend:"${HEAD_SHA}" \ - --profile frontend - - - name: Push deployable backend runtime image - id: deployable_backend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - - DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" - DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" - - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" - docker push "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null - - DEPLOYABLE_BACKEND_DIGEST_REF="$({ - docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ - | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ - | head -n 1 - } || true)" - - if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve deployable backend digest reference" - exit 1 - fi - - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" - - - name: Build and push complete CICD image - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - run: | - set -e - umask 077 - trap 'rm -f /tmp/ssh_key' EXIT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - - if [ -n "${BASE_HASH_INPUT}" ]; then - BASE_HASH="${BASE_HASH_INPUT}" - echo "Using provided immutable base hash from upstream workflow dispatch: ${BASE_HASH}" - else - BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) - echo "No base_hash input provided; computed base hash locally: ${BASE_HASH}" - fi - - BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - - verify_base_image() { - # Fast path: base image already present locally. - if docker image inspect "${BASE_IMAGE}" >/dev/null 2>&1; then - return 0 - fi - - # Manifest inspect unreliable on these runners; use pull as truth. - timeout 1800 docker pull "${BASE_IMAGE}" >/tmp/base-image-pull.log 2>&1 - } - - max_attempts=2 - sleep_seconds=15 - for i in $(seq 1 "${max_attempts}"); do - echo "Base availability check ${i}/${max_attempts}..." - if verify_base_image; then - echo "✓ Base image available and hash matched: ${BASE_IMAGE}" - break - else - pull_exit=$? - echo "Base pull attempt ${i} failed with exit code ${pull_exit}" - if [ "${pull_exit}" -eq 124 ]; then - echo "Pull timed out after 1800s while downloading base image" - fi - if [ -s /tmp/base-image-pull.log ]; then - tail -n 40 /tmp/base-image-pull.log || true - fi - - if [ "${i}" -eq "${max_attempts}" ]; then - echo "❌ Required immutable base image is not available or mismatched: ${BASE_IMAGE}" - exit 1 - fi - sleep "${sleep_seconds}" - fi - done - - echo "✓ Base image ready: ${BASE_IMAGE}" - - echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key - chmod 600 /tmp/ssh_key - export DOCKER_BUILDKIT=1 - - docker build -f Dockerfile.cicd \ - --secret id=ssh_private_key,src=/tmp/ssh_key \ - --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \ - --build-arg GITHUB_SHA="${HEAD_SHA}" \ - --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ - -t cicd:latest . - - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" - - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" - - - *failure_diagnostics_step - - dispatch-tests: - name: Dispatch CICD Tests - runs-on: ubuntu-act - timeout-minutes: 15 - needs: build - if: needs.build.result == 'success' - steps: - - name: Dispatch tests workflow - env: - ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - HEAD_SHA: ${{ needs.build.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }} - REPO_FULL: ${{ github.repository }} - HEAD_REF: ${{ github.head_ref }} - REF_NAME: ${{ github.ref_name }} - TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }} - run: | - set -e - - DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}" - - if [ -z "${DISPATCH_TOKEN}" ]; then - echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions." - exit 1 - fi - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - REPO_OWNER="${REPO_FULL%/*}" - REPO_NAME="${REPO_FULL#*/}" - TARGET_REF="${HEAD_REF:-${REF_NAME}}" - TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}" - - echo "trace_id=${TRACE_ID}" - echo "target_ref=${TARGET_REF}" - - CANDIDATE_API_BASES=() - if [ -n "${GITHUB_SERVER_URL:-}" ]; then - CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") - fi - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") - CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - - ensure_curl() { - if command -v curl >/dev/null 2>&1; then - return 0 - fi - if command -v apt-get >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get install -y -qq curl ca-certificates - fi - command -v curl >/dev/null 2>&1 - } - - ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; } - - HELPER_PATH="/tmp/dispatch-workflow.sh" - - fetch_dispatch_helper() { - local helper_ref="$1" - local api_base - for api_base in "${CANDIDATE_API_BASES[@]}"; do - helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" - if curl -fsS --connect-timeout 5 --max-time 20 \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "User-Agent: plex-playlist-cicd-main" \ - -o "${HELPER_PATH}" \ - "${helper_url}"; then - chmod +x "${HELPER_PATH}" - return 0 - fi - done - return 1 - } - - if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then - echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" - exit 1 - fi - - DISPATCH_ARGS=( - --token "${DISPATCH_TOKEN}" - --repo "${REPO_FULL}" - --workflow "cicd-tests.yaml" - --ref "${TARGET_REF}" - --head-sha "${HEAD_SHA}" - --source-workflow "CICD Main Build" - --trace-id "${TRACE_ID}" - --input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" - --input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" - ) - - for API_BASE in "${CANDIDATE_API_BASES[@]}"; do - DISPATCH_ARGS+=(--api-base "${API_BASE}") - done - - "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" - - - *failure_diagnostics_step diff --git a/Dockerfile.backend b/Dockerfile.backend index fdc12fa..12a6015 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -1,37 +1,30 @@ # Backend Dockerfile for FastAPI with Python 3.14 -FROM python:3.14-slim +FROM python:3.14-slim AS dependencies -# Set working directory WORKDIR /app -# Install system dependencies -RUN apt-get update && apt-get install -y \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv -# Create and activate virtual environment ENV VIRTUAL_ENV=/app/.venv -RUN uv venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# Copy dependency files first for better caching COPY backend/pyproject.toml backend/uv.lock* ./ # Hatchling resolves the readme path ../README.md relative to the backend # package root (/app). Create a stub so metadata validation succeeds without # requiring the file to pass through .dockerignore. RUN echo '# plex-playlist' > /README.md +RUN uv venv "$VIRTUAL_ENV" && uv sync --frozen --no-dev -# Install runtime dependencies only -RUN uv sync --frozen --no-dev +FROM python:3.14-slim AS runtime -# Copy application code -COPY backend/ . +WORKDIR /app + +ENV VIRTUAL_ENV=/app/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY --from=dependencies /app/.venv /app/.venv +COPY backend/src ./src -# Expose port EXPOSE 8000 -# Default command - can be overridden in compose for development CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Dockerfile.frontend b/Dockerfile.frontend index be5d130..94897d0 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -1,40 +1,20 @@ # Frontend Dockerfile for Vue/Vite TypeScript project -FROM node:20-alpine AS base +FROM node:24-alpine AS build -# Set working directory WORKDIR /app -# Copy package files first for better caching -COPY frontend/package*.json ./ -COPY frontend/.yarnrc.yml ./ -COPY frontend/yarn.lock* frontend/pnpm-lock.yaml* ./ +COPY frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml ./ +COPY frontend/.yarn/ ./.yarn/ +RUN corepack enable && yarn install --immutable -# Install dependencies -RUN if [ -f yarn.lock ]; then corepack enable && corepack yarn install; \ - elif [ -f pnpm-lock.yaml ]; then npm install -g pnpm && pnpm install; \ - else npm install; fi - -# Copy application code COPY frontend/ . +RUN yarn build -# Development stage -FROM base AS development -EXPOSE 5173 -CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] - -# Production build stage -FROM base AS build -RUN if [ -f yarn.lock ]; then corepack yarn build; \ - elif [ -f pnpm-lock.yaml ]; then pnpm build; \ - else npm run build; fi - -# Production stage FROM nginx:alpine AS production -# Copy built assets from build stage + COPY --from=build /app/dist /usr/share/nginx/html -# Copy nginx configuration COPY frontend/nginx.conf /etc/nginx/nginx.conf -# Expose port + EXPOSE 80 -# Start nginx + CMD ["nginx", "-g", "daemon off;"] diff --git a/backend/tests/integration/test_runtime_blackbox.py b/backend/tests/integration/test_runtime_blackbox.py new file mode 100644 index 0000000..4d75810 --- /dev/null +++ b/backend/tests/integration/test_runtime_blackbox.py @@ -0,0 +1,55 @@ +"""Runtime black-box integration tests executed over the container network.""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request + +import pytest + +BACKEND_BASE_URL = os.getenv("INTEGRATION_BACKEND_URL", "http://backend:8000").rstrip( + "/" +) + + +def _fetch(path: str) -> tuple[int, str]: + """Fetch an endpoint and return status code and body text.""" + url = f"{BACKEND_BASE_URL}{path}" + request = urllib.request.Request(url=url, method="GET") + try: + with urllib.request.urlopen(request, timeout=5) as response: + body = response.read().decode("utf-8", errors="replace") + return response.status, body + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + return exc.code, body + + +@pytest.mark.integration +def test_root_endpoint_runtime() -> None: + """Runtime root endpoint should return backend API message.""" + status, body = _fetch("/") + assert status == 200 + payload = json.loads(body) + assert payload == {"message": "Plex Playlist Backend API"} + + +@pytest.mark.integration +def test_health_endpoint_runtime() -> None: + """Runtime health endpoint should report healthy database connectivity.""" + status, body = _fetch("/health") + assert status == 200 + payload = json.loads(body) + assert payload.get("status") == "healthy" + assert payload.get("database") == "connected" + + +@pytest.mark.integration +def test_compatibility_endpoint_runtime() -> None: + """Runtime compatibility endpoint should report policy check success.""" + status, body = _fetch("/compatibility") + assert status == 200 + payload = json.loads(body) + assert payload.get("ok") is True diff --git a/docs/CICD_MULTI_STAGE_BUILD.md b/docs/CICD_MULTI_STAGE_BUILD.md index 9f1691d..6092969 100644 --- a/docs/CICD_MULTI_STAGE_BUILD.md +++ b/docs/CICD_MULTI_STAGE_BUILD.md @@ -142,14 +142,10 @@ jobs: ### Responsibility Split -- `.gitea/workflows/cicd-start.yaml` owns startup routing and trace propagation. -- `.gitea/workflows/cicd-source-checks.yaml` owns early source-level - format/lint/type gating before any promotion dispatch. -- `.gitea/workflows/docker-build-base.yaml` owns base publication and verification. -- `.gitea/workflows/docker-build-main.yaml` owns complete-image publication. -- `.gitea/workflows/cicd-checks.yaml` and `.gitea/workflows/cicd-tests.yaml` - own post-build CI validation and tests. -- Main CI never rebuilds the base image locally. +- `.gitea/workflows/cicd.yaml` owns the full CI pipeline: base image publish, + CICD image publish, source checks, unit tests, runtime image publication, + integration tests, and E2E tests. +- The older split workflows remain only as deprecated/manual helper paths. ### Runtime Boundary Enforcement @@ -173,7 +169,7 @@ and deployable runtime artifacts before publishing the complete CICD image. Workflow location: -- `.gitea/workflows/docker-build-main.yaml` +- `.gitea/workflows/cicd.yaml` - `Verify deployable runtime boundaries` - `Build and verify deployable runtime image purity` diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 0c33cd2..b9caea0 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -30,6 +30,7 @@ Excluded: ### Runtime Artifact Definition - Container build source: `Dockerfile.backend`. +- Build shape: two-stage build with a dependency stage and a minimal runtime stage. - Runtime base image: `python:3.14-slim`. - Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`. - Exposed runtime port: `8000`. @@ -79,6 +80,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth. ### Frontend Runtime Artifact Definition - Container build source: `Dockerfile.frontend` (target `production`). +- Build shape: two-stage build with a dependency/build stage and a minimal nginx runtime stage. - Runtime base image: `nginx:alpine`. - Runtime process: `nginx -g "daemon off;"`. - Exposed runtime port: `80`. @@ -151,14 +153,14 @@ Current enforcement implemented in CI: - Dockerfile target boundary checks: - Script: `scripts/check-dockerfile-boundaries.sh` - - Workflow: `.gitea/workflows/docker-build-main.yaml` + - Workflow: `.gitea/workflows/cicd.yaml` - Deployable runtime image purity checks: - Script: `scripts/verify-deployable-image-purity.sh` - - Workflow: `.gitea/workflows/docker-build-main.yaml` + - Workflow: `.gitea/workflows/cicd.yaml` - Checks include binary presence and profile-specific package metadata probes to detect CI/development tooling leakage. - Runtime black-box integration checks against deployed backend container: - - Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job) + - Workflow: `.gitea/workflows/cicd.yaml` (`integration-tests` job) - Inputs: deployable backend commit tag reference and immutable digest reference from main build dispatch. - Assertions: digest/tag consistency and live endpoint behavior for `/`, diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d528284..82a4f75 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -36,7 +36,7 @@ Scope boundary: - Contract definition lives in `DEPLOYABLE_RUNTIME_CONTRACT.md`. - CI enforcement now includes Dockerfile boundary checks and deployable image - purity checks in `.gitea/workflows/docker-build-main.yaml`. + purity checks in `.gitea/workflows/cicd.yaml`. - Broader workflow redesign and deployment wiring remain out of scope for this repository's runtime contract document and belong to follow-up work under epic #66. @@ -60,8 +60,8 @@ Automated checks: and fails if CI/development binaries or package metadata artifacts are present. -These checks run in `.gitea/workflows/docker-build-main.yaml` before publishing -the complete CICD image. +These checks run in `.gitea/workflows/cicd.yaml` before publishing the +complete CICD image. ## Quick Start diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 1e05aeb..4f165a4 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -1,5 +1,10 @@ import { defineConfig, devices } from '@playwright/test'; +const runtimeBaseURL = process.env.PLAYWRIGHT_BASE_URL; +const useRuntimeServices = Boolean(runtimeBaseURL); +const junitOutputFile = process.env.PLAYWRIGHT_JUNIT_OUTPUT_FILE ?? 'playwright-results.xml'; +const outputDir = process.env.PLAYWRIGHT_OUTPUT_DIR ?? 'playwright-artifacts'; + export default defineConfig({ testDir: './tests/e2e', timeout: process.env.CI ? 60 * 1000 : 30 * 1000, // Longer timeout in CI @@ -10,11 +15,10 @@ export default defineConfig({ forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, workers: process.env.CI ? 1 : undefined, - reporter: process.env.CI - ? [['list'], ['junit', { outputFile: 'playwright-results.xml' }]] - : 'html', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: junitOutputFile }]] : 'html', + outputDir, use: { - baseURL: 'http://localhost:5173', + baseURL: runtimeBaseURL ?? 'http://localhost:5173', trace: 'on-first-retry', headless: process.env.CI ? true : false, // CI-specific browser optimizations @@ -78,12 +82,16 @@ export default defineConfig({ }, }, ], - webServer: { - command: 'yarn dev --host 0.0.0.0 --port 5173 --strictPort', - url: 'http://localhost:5173', - reuseExistingServer: !process.env.CI, - timeout: process.env.CI ? 180 * 1000 : 120 * 1000, // Longer startup timeout in CI - stderr: 'pipe', - stdout: 'pipe', - }, + ...(useRuntimeServices + ? {} + : { + webServer: { + command: 'yarn dev --host 0.0.0.0 --port 5173 --strictPort', + url: 'http://localhost:5173', + reuseExistingServer: !process.env.CI, + timeout: process.env.CI ? 180 * 1000 : 120 * 1000, // Longer startup timeout in CI + stderr: 'pipe', + stdout: 'pipe', + }, + }), }); -- 2.49.1 From fe02644e1ddd86acbb2b42adeadc990dcecd51d7 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 11:48:38 -0400 Subject: [PATCH 02/80] Trigger CICD on every push --- .gitea/workflows/cicd.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 6d4a551..386cb08 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2,7 +2,6 @@ name: CICD on: push: - branches: ["**"] pull_request: branches: [main, develop] workflow_dispatch: -- 2.49.1 From 9a43d9090b977d96739643628c650fe86715760e Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 13:01:27 -0400 Subject: [PATCH 03/80] Reclaim runner disk before CICD image build --- .gitea/workflows/cicd.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 386cb08..6493bc6 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -239,6 +239,17 @@ jobs: umask 077 trap 'rm -f /tmp/ssh_key' EXIT + echo "=== Pre-build disk telemetry ===" + df -h || true + if command -v docker >/dev/null 2>&1; then + docker system df || true + docker builder prune -af || true + docker system prune -af --volumes || true + echo "=== Post-prune disk telemetry ===" + df -h || true + docker system df || true + fi + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin BASE_HASH="${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}" -- 2.49.1 From 0791f9257e78739d52ead345b589a1af8ca27f5c Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 16:45:05 -0400 Subject: [PATCH 04/80] Keep integration tests out of backend unit lane --- .gitea/workflows/cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 6493bc6..9af9ecc 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -339,7 +339,7 @@ jobs: docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " cd /workspace/backend && source .venv/bin/activate && - uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95 + uv run pytest -v --tb=short --ignore=tests/integration --cov=src --cov-report=term-missing --cov-fail-under=95 " 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} -- 2.49.1 From 4af111fcdb780882aa5883142c1c1d9c18c95873 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 16:51:39 -0400 Subject: [PATCH 05/80] Align CI and Poe test lanes with runtime compose flows --- .gitea/workflows/cicd.yaml | 4 +- backend/pyproject.toml | 8 ++-- backend/tests/integration/test_api.py | 1 - .../integration/test_runtime_blackbox.py | 4 +- compose.ci.e2e.yml | 45 +++++++++++++++++++ compose.ci.integration.yml | 38 ++++++++++++++++ scripts/run-ci-e2e-compose.sh | 29 ++++++++++++ scripts/run-ci-integration-compose.sh | 23 ++++++++++ 8 files changed, 142 insertions(+), 10 deletions(-) create mode 100644 compose.ci.e2e.yml create mode 100644 compose.ci.integration.yml create mode 100644 scripts/run-ci-e2e-compose.sh create mode 100644 scripts/run-ci-integration-compose.sh diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 9af9ecc..af2016b 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -339,7 +339,7 @@ jobs: docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " cd /workspace/backend && source .venv/bin/activate && - uv run pytest -v --tb=short --ignore=tests/integration --cov=src --cov-report=term-missing --cov-fail-under=95 + uv run pytest -v --tb=short -m 'not integration' --cov=src --cov-report=term-missing --cov-fail-under=95 " 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} @@ -842,7 +842,7 @@ jobs: fi docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm integration-tests \ - bash -c "cd /workspace/backend && source .venv/bin/activate && uv run pytest -v --tb=short tests/integration/test_runtime_blackbox.py" + bash -c "cd /workspace/backend && source .venv/bin/activate && uv run pytest -v --tb=short -m integration tests/integration/" echo "✅ Runtime integration checks passed" } 2>&1 | tee "${LOG_FILE}" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 917c238..1e05697 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -166,15 +166,15 @@ setup = [ ] test-all = ["test-backend", "test-frontend", "test-integration"] # === Testing Tasks === -test-backend = {shell = "cd backend && uv run pytest", help = "Run backend unit tests"} -test-backend-cov = {shell = "cd backend && uv run pytest --cov", help = "Run backend tests with coverage"} -test-e2e = {shell = "cd frontend && yarn test:e2e", help = "Run end-to-end tests"} +test-backend = {shell = "cd backend && uv run pytest -m 'not integration'", help = "Run backend unit tests (excludes integration marker)"} +test-backend-cov = {shell = "cd backend && uv run pytest -m 'not integration' --cov", help = "Run backend unit tests with coverage (excludes integration marker)"} +test-e2e = {shell = "bash ./scripts/run-ci-e2e-compose.sh", help = "Run E2E tests via compose against deployable runtime images"} test-frontend = {shell = "cd frontend && yarn test", help = "Run frontend unit tests"} test-frontend-cov = {shell = "cd frontend && yarn test:coverage", help = "Run frontend tests with coverage"} test-full = ["test-backend-cov", "test-frontend-cov", "test-integration", "test-e2e"] # === Smart Conditional Tasks === test-if-changed = {shell = "if git diff --quiet HEAD~1 backend/ frontend/; then echo 'No changes detected, skipping tests'; else poe test-unit; fi", help = "Only run tests if code has changed"} -test-integration = {shell = "cd backend && uv run pytest tests/integration/", help = "Run backend integration tests"} +test-integration = {shell = "bash ./scripts/run-ci-integration-compose.sh", help = "Run integration tests via compose against deployable runtime image"} test-parallel = {shell = "poe test-backend & poe test-frontend & wait", help = "Run unit tests in parallel for speed"} # === Comprehensive Testing === test-unit = ["test-backend", "test-frontend"] diff --git a/backend/tests/integration/test_api.py b/backend/tests/integration/test_api.py index 176ea23..3f22b99 100644 --- a/backend/tests/integration/test_api.py +++ b/backend/tests/integration/test_api.py @@ -14,7 +14,6 @@ from backend.main import app, compatibility_status, get_api_session client = TestClient(app) -@pytest.mark.integration class TestAPIIntegration: """Integration tests for API endpoints.""" diff --git a/backend/tests/integration/test_runtime_blackbox.py b/backend/tests/integration/test_runtime_blackbox.py index 4d75810..7919961 100644 --- a/backend/tests/integration/test_runtime_blackbox.py +++ b/backend/tests/integration/test_runtime_blackbox.py @@ -12,6 +12,7 @@ import pytest BACKEND_BASE_URL = os.getenv("INTEGRATION_BACKEND_URL", "http://backend:8000").rstrip( "/" ) +pytestmark = pytest.mark.integration def _fetch(path: str) -> tuple[int, str]: @@ -27,7 +28,6 @@ def _fetch(path: str) -> tuple[int, str]: return exc.code, body -@pytest.mark.integration def test_root_endpoint_runtime() -> None: """Runtime root endpoint should return backend API message.""" status, body = _fetch("/") @@ -36,7 +36,6 @@ def test_root_endpoint_runtime() -> None: assert payload == {"message": "Plex Playlist Backend API"} -@pytest.mark.integration def test_health_endpoint_runtime() -> None: """Runtime health endpoint should report healthy database connectivity.""" status, body = _fetch("/health") @@ -46,7 +45,6 @@ def test_health_endpoint_runtime() -> None: assert payload.get("database") == "connected" -@pytest.mark.integration def test_compatibility_endpoint_runtime() -> None: """Runtime compatibility endpoint should report policy check success.""" status, body = _fetch("/compatibility") diff --git a/compose.ci.e2e.yml b/compose.ci.e2e.yml new file mode 100644 index 0000000..4655847 --- /dev/null +++ b/compose.ci.e2e.yml @@ -0,0 +1,45 @@ +--- +services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD:-plex_password} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local} + environment: + DATABASE_URL: ${DB_URL:-postgresql://plex_user:plex_password@database:5432/plex_playlist} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + + frontend: + image: ${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local} + depends_on: + backend: + condition: service_started + + e2e-tests: + image: ${CICD_IMAGE:-cicd:local} + depends_on: + frontend: + condition: service_started + environment: + CI: "true" + PLAYWRIGHT_BASE_URL: http://frontend:80 + PLAYWRIGHT_OUTPUT_DIR: /tmp/playwright-artifacts + PLAYWRIGHT_JUNIT_OUTPUT_FILE: /tmp/playwright-artifacts/playwright-results.xml + volumes: + - ${E2E_ARTIFACT_DIR:-./frontend/.playwright-artifacts}:/tmp/playwright-artifacts + command: + - bash + - -lc + - cd /workspace/frontend && yarn test:e2e --reporter=list --timeout=90000 diff --git a/compose.ci.integration.yml b/compose.ci.integration.yml new file mode 100644 index 0000000..94ec3d1 --- /dev/null +++ b/compose.ci.integration.yml @@ -0,0 +1,38 @@ +--- +services: + database: + image: postgres:16-alpine + environment: + POSTGRES_DB: plex_playlist + POSTGRES_USER: plex_user + POSTGRES_PASSWORD: ${DB_PASSWORD:-plex_password} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"] + interval: 5s + timeout: 3s + retries: 12 + + backend: + image: ${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local} + environment: + DATABASE_URL: ${DB_URL:-postgresql://plex_user:plex_password@database:5432/plex_playlist} + ENVIRONMENT: production + depends_on: + database: + condition: service_healthy + + integration-tests: + image: ${CICD_IMAGE:-cicd:local} + depends_on: + backend: + condition: service_started + environment: + CI: "true" + INTEGRATION_BACKEND_URL: http://backend:8000 + command: + - bash + - -lc + - >- + cd /workspace/backend && + source .venv/bin/activate && + uv run pytest -v --tb=short -m integration tests/integration/ diff --git a/scripts/run-ci-e2e-compose.sh b/scripts/run-ci-e2e-compose.sh new file mode 100644 index 0000000..98b1aec --- /dev/null +++ b/scripts/run-ci-e2e-compose.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/compose.ci.e2e.yml" +PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-e2e-local}" + +mkdir -p "${ROOT_DIR}/frontend/.playwright-artifacts" + +export CICD_IMAGE="${CICD_IMAGE:-cicd:local}" +export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}" +export DEPLOYABLE_FRONTEND_IMAGE="${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local}" +export DB_PASSWORD="${DB_PASSWORD:-plex_password}" +export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}" +export E2E_ARTIFACT_DIR="${E2E_ARTIFACT_DIR:-${ROOT_DIR}/frontend/.playwright-artifacts}" + +cleanup() { + docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +echo "Running E2E tests against runtime backend/frontend images via compose" +echo "CICD_IMAGE=${CICD_IMAGE}" +echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}" +echo "DEPLOYABLE_FRONTEND_IMAGE=${DEPLOYABLE_FRONTEND_IMAGE}" +echo "E2E_ARTIFACT_DIR=${E2E_ARTIFACT_DIR}" + +docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up --abort-on-container-exit --exit-code-from e2e-tests e2e-tests diff --git a/scripts/run-ci-integration-compose.sh b/scripts/run-ci-integration-compose.sh new file mode 100644 index 0000000..418d7f1 --- /dev/null +++ b/scripts/run-ci-integration-compose.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/compose.ci.integration.yml" +PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-int-local}" + +export CICD_IMAGE="${CICD_IMAGE:-cicd:local}" +export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}" +export DB_PASSWORD="${DB_PASSWORD:-plex_password}" +export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}" + +cleanup() { + docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +echo "Running integration tests against runtime backend image via compose" +echo "CICD_IMAGE=${CICD_IMAGE}" +echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}" + +docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up --abort-on-container-exit --exit-code-from integration-tests integration-tests -- 2.49.1 From 2df9dbe7a09d2b176477455db5dc8e8a20936ca3 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 17:24:44 -0400 Subject: [PATCH 06/80] ci: retrigger workflow run attempt 2 -- 2.49.1 From 4855be1c12bc67f1ca4dad10a640e788ba4562b5 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 18:06:51 -0400 Subject: [PATCH 07/80] ci: retrigger workflow run attempt 2 -- 2.49.1 From 6f3d96d6ef4e58663294dc4933dc1dd0d423703d Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 18:59:05 -0400 Subject: [PATCH 08/80] Handle missing frontend .yarn directory in runtime build --- Dockerfile.frontend | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile.frontend b/Dockerfile.frontend index 94897d0..b66cb41 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -4,7 +4,6 @@ FROM node:24-alpine AS build WORKDIR /app COPY frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml ./ -COPY frontend/.yarn/ ./.yarn/ RUN corepack enable && yarn install --immutable COPY frontend/ . -- 2.49.1 From 6963c3ae2f6ca05c823b24b9b764156db234ac07 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 19:47:50 -0400 Subject: [PATCH 09/80] ci: retrigger workflow run attempt 2 -- 2.49.1 From 738ab39736653ab2a6535b94fd815f9966d1d5c3 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 22:02:18 -0400 Subject: [PATCH 10/80] Resolve CICD image tag for runtime-based jobs --- .gitea/workflows/cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index af2016b..ceb2470 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -302,7 +302,7 @@ jobs: - &ensure_cicd_image_step name: Ensure CICD image is available env: - HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || needs.runtime_images.outputs.head_sha || github.sha }} run: | IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" if docker image inspect "${IMAGE}" >/dev/null 2>&1; then -- 2.49.1 From 08d36e958bb1269fa640d351b48b54d38aa6de29 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 6 Jul 2026 23:12:38 -0400 Subject: [PATCH 11/80] Run integration and E2E against deployable app images --- .gitea/workflows/cicd.yaml | 65 ++++++++++----------- compose.ci.e2e.yml | 17 ------ compose.ci.integration.yml | 16 ------ scripts/run-ci-e2e-compose.sh | 15 ++++- scripts/run-ci-integration-compose.sh | 81 +++++++++++++++++++++++++-- 5 files changed, 120 insertions(+), 74 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index ceb2470..8e7e503 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -647,7 +647,6 @@ jobs: COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}" DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" - CICD_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" if ! command -v docker >/dev/null 2>&1; then echo "❌ Docker is required" @@ -680,15 +679,6 @@ jobs: depends_on: database: condition: service_healthy - - integration-tests: - image: ${CICD_IMAGE} - depends_on: - backend: - condition: service_started - environment: - CI: "true" - INTEGRATION_BACKEND_URL: http://backend:8000 EOF fetch_backend_endpoint() { @@ -749,7 +739,6 @@ jobs: echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - docker pull "${CICD_IMAGE}" docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" @@ -776,7 +765,7 @@ jobs: echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" - export DB_PASSWORD DB_URL CICD_IMAGE DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}" + export DB_PASSWORD DB_URL DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}" docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend @@ -841,9 +830,6 @@ jobs: exit 1 fi - docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm integration-tests \ - bash -c "cd /workspace/backend && source .venv/bin/activate && uv run pytest -v --tb=short -m integration tests/integration/" - echo "✅ Runtime integration checks passed" } 2>&1 | tee "${LOG_FILE}" @@ -877,10 +863,10 @@ jobs: echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - *configure_registry_host_step - - *ensure_cicd_image_step - name: Run E2E tests against runtime services via compose env: HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_frontend_tag_ref }} @@ -896,7 +882,7 @@ jobs: COMPOSE_PROJECT_NAME="plex-e2e-${RUN_ID}-${RUN_ATTEMPT}" DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" - CICD_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + E2E_REPO_DIR="$(mktemp -d)" FRONTEND_URL_RUNNER="http://frontend:80" if ! command -v docker >/dev/null 2>&1; then @@ -936,19 +922,6 @@ jobs: depends_on: backend: condition: service_started - - e2e-tests: - image: ${CICD_IMAGE} - depends_on: - frontend: - condition: service_started - environment: - CI: "true" - PLAYWRIGHT_BASE_URL: http://frontend:80 - PLAYWRIGHT_OUTPUT_DIR: /tmp/playwright-artifacts - PLAYWRIGHT_JUNIT_OUTPUT_FILE: /tmp/playwright-artifacts/playwright-results.xml - volumes: - - ${E2E_ARTIFACT_DIR}:/tmp/playwright-artifacts EOF fetch_http() { @@ -1032,6 +1005,8 @@ jobs: cleanup() { docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true + rm -rf "${E2E_REPO_DIR}" >/dev/null 2>&1 || true + rm -f ~/.ssh/id_rsa >/dev/null 2>&1 || true } trap cleanup EXIT @@ -1041,8 +1016,22 @@ jobs: exit 1 fi + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" "${E2E_REPO_DIR}" + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git -C "${E2E_REPO_DIR}" fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git -C "${E2E_REPO_DIR}" checkout FETCH_HEAD -- frontend + else + git -C "${E2E_REPO_DIR}" checkout HEAD -- frontend + fi + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - docker pull "${CICD_IMAGE}" docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" docker pull "${DEPLOYABLE_FRONTEND_TAG_REF}" @@ -1091,7 +1080,7 @@ jobs: echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}" echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}" - export DB_PASSWORD DB_URL CICD_IMAGE E2E_ARTIFACT_DIR="${ARTIFACT_DIR}" + export DB_PASSWORD DB_URL E2E_ARTIFACT_DIR="${ARTIFACT_DIR}" export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}" export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}" @@ -1161,8 +1150,16 @@ jobs: exit 1 fi - docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm e2e-tests \ - bash -c "cd /workspace/frontend && yarn test:e2e --reporter=list --timeout=90000" \ + docker run --rm \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + -e CI=true \ + -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ + -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ + -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ + -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \ + -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ + mcr.microsoft/playwright:v1.56.1-jammy \ + bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \ 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} diff --git a/compose.ci.e2e.yml b/compose.ci.e2e.yml index 4655847..9e98761 100644 --- a/compose.ci.e2e.yml +++ b/compose.ci.e2e.yml @@ -26,20 +26,3 @@ services: depends_on: backend: condition: service_started - - e2e-tests: - image: ${CICD_IMAGE:-cicd:local} - depends_on: - frontend: - condition: service_started - environment: - CI: "true" - PLAYWRIGHT_BASE_URL: http://frontend:80 - PLAYWRIGHT_OUTPUT_DIR: /tmp/playwright-artifacts - PLAYWRIGHT_JUNIT_OUTPUT_FILE: /tmp/playwright-artifacts/playwright-results.xml - volumes: - - ${E2E_ARTIFACT_DIR:-./frontend/.playwright-artifacts}:/tmp/playwright-artifacts - command: - - bash - - -lc - - cd /workspace/frontend && yarn test:e2e --reporter=list --timeout=90000 diff --git a/compose.ci.integration.yml b/compose.ci.integration.yml index 94ec3d1..cbbfc8d 100644 --- a/compose.ci.integration.yml +++ b/compose.ci.integration.yml @@ -20,19 +20,3 @@ services: depends_on: database: condition: service_healthy - - integration-tests: - image: ${CICD_IMAGE:-cicd:local} - depends_on: - backend: - condition: service_started - environment: - CI: "true" - INTEGRATION_BACKEND_URL: http://backend:8000 - command: - - bash - - -lc - - >- - cd /workspace/backend && - source .venv/bin/activate && - uv run pytest -v --tb=short -m integration tests/integration/ diff --git a/scripts/run-ci-e2e-compose.sh b/scripts/run-ci-e2e-compose.sh index 98b1aec..bf0e55b 100644 --- a/scripts/run-ci-e2e-compose.sh +++ b/scripts/run-ci-e2e-compose.sh @@ -7,7 +7,6 @@ PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-e2e-local}" mkdir -p "${ROOT_DIR}/frontend/.playwright-artifacts" -export CICD_IMAGE="${CICD_IMAGE:-cicd:local}" export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}" export DEPLOYABLE_FRONTEND_IMAGE="${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local}" export DB_PASSWORD="${DB_PASSWORD:-plex_password}" @@ -21,9 +20,19 @@ cleanup() { trap cleanup EXIT echo "Running E2E tests against runtime backend/frontend images via compose" -echo "CICD_IMAGE=${CICD_IMAGE}" echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}" echo "DEPLOYABLE_FRONTEND_IMAGE=${DEPLOYABLE_FRONTEND_IMAGE}" echo "E2E_ARTIFACT_DIR=${E2E_ARTIFACT_DIR}" -docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up --abort-on-container-exit --exit-code-from e2e-tests e2e-tests +docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend + +docker run --rm \ + --network "${PROJECT_NAME}_default" \ + -e CI=true \ + -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ + -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ + -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ + -v "${ROOT_DIR}/frontend:/workspace/frontend" \ + -v "${E2E_ARTIFACT_DIR}:/tmp/playwright-artifacts" \ + mcr.microsoft/playwright:v1.56.1-jammy \ + bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" diff --git a/scripts/run-ci-integration-compose.sh b/scripts/run-ci-integration-compose.sh index 418d7f1..b5a3e46 100644 --- a/scripts/run-ci-integration-compose.sh +++ b/scripts/run-ci-integration-compose.sh @@ -5,7 +5,6 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" COMPOSE_FILE="${ROOT_DIR}/compose.ci.integration.yml" PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-int-local}" -export CICD_IMAGE="${CICD_IMAGE:-cicd:local}" export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}" export DB_PASSWORD="${DB_PASSWORD:-plex_password}" export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}" @@ -16,8 +15,82 @@ cleanup() { trap cleanup EXIT -echo "Running integration tests against runtime backend image via compose" -echo "CICD_IMAGE=${CICD_IMAGE}" +echo "Running runtime integration checks against deployable backend image via compose" echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}" -docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up --abort-on-container-exit --exit-code-from integration-tests integration-tests +docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend + +fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")' + )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null + then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" +} + +backend_ready=false +for i in $(seq 1 40); do + backend_running_count="$(docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)" + if [ "${backend_running_count}" -eq 0 ]; then + echo "❌ Backend container exited before becoming healthy" + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 +done + +if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + exit 1 +fi + +root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" +if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + exit 1 +fi + +compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" +if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + exit 1 +fi + +echo "✅ Runtime integration checks passed" -- 2.49.1 From 71b9dda8a4f216a37fa5f17b108a457a8c1ed733 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 07:12:34 -0400 Subject: [PATCH 12/80] ci: retrigger workflow run -- 2.49.1 From 8619e00a17a7b760e9a525e7e7c190d9e9f46547 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 08:14:10 -0400 Subject: [PATCH 13/80] Add runner diagnostics and rebalance source lanes --- .gitea/workflows/cicd.yaml | 10 ++++++---- backend/pyproject.toml | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 8e7e503..50243a4 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -354,8 +354,9 @@ jobs: precommit-tests: name: Pre-commit Checks - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act + # Pin one source-check lane to the 8GB pool so source checks can fan out + # across both runner pools instead of serializing behind a single label. + runs-on: ubuntu-act-8gb timeout-minutes: 30 needs: build_cicd steps: @@ -400,8 +401,9 @@ jobs: frontend-tests: name: Frontend Tests - # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act + # Pin one source-check lane to the 8GB pool so source checks can fan out + # across both runner pools instead of serializing behind a single label. + runs-on: ubuntu-act-8gb timeout-minutes: 25 needs: build_cicd steps: diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1e05697..8204bac 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -154,6 +154,8 @@ reset = [ {shell = "cd frontend && rm -rf node_modules 2>/dev/null || true"}, "deps-install" ] +runner-diagnostics = {shell = "xonsh ./scripts/gitea-actions/collect_runner_diagnostics.xsh", help = "Collect recent runner, docker, and system diagnostics from all known runner hosts"} +runner-discover = {shell = "xonsh ./scripts/gitea-actions/discover-runners.xsh --batch", help = "Collect runner inventory, labels, and baseline health from all known runner hosts"} # === Development Setup (New Developer Onboarding) === setup = [ "deps-install", -- 2.49.1 From d59217912d0aaa0f0b36417852a45a9b3e1a4c53 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 08:19:15 -0400 Subject: [PATCH 14/80] Discover actual runner containers dynamically --- scripts/gitea-actions/discover-runners.xsh | 70 ++++++++++++---------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/scripts/gitea-actions/discover-runners.xsh b/scripts/gitea-actions/discover-runners.xsh index 137963c..5475f92 100755 --- a/scripts/gitea-actions/discover-runners.xsh +++ b/scripts/gitea-actions/discover-runners.xsh @@ -41,6 +41,17 @@ def first_existing_compose(host, root): return None +def list_runner_containers(host): + """Return actual runner container names present on a host.""" + rc, text = ssh( + host, + "docker ps -a --format '{{.Names}}' | grep '^gitea-act-runner-' || true", + ) + if rc != 0 or not text: + return [] + return [line.strip() for line in text.splitlines() if line.strip()] + + report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}" mkdir -p @(report_dir) report_path = os.path.join(report_dir, "discovery-report.txt") @@ -84,31 +95,29 @@ for host, root in HOSTS: _, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'") data["container_sizes"] = size_text - # Explicit runner names avoid label-filter parsing issues seen previously. - rc1, inspect_1 = ssh( - host, - "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'", - ) - rc2, inspect_2 = ssh( - host, - "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'", - ) - if rc1 != 0: - inspect_1 = "RUNNER1_NOT_FOUND" - if rc2 != 0: - inspect_2 = "RUNNER2_NOT_FOUND" - data["runner_inspect"] = (inspect_1 + "\n" + inspect_2).strip() + runner_containers = list_runner_containers(host) + data["runner_containers"] = runner_containers - log1_rc, log1 = ssh(host, "docker logs --tail=40 gitea-act-runner-1") - log2_rc, log2 = ssh(host, "docker logs --tail=40 gitea-act-runner-2") - if log1_rc != 0: - log1 = "RUNNER1_LOGS_UNAVAILABLE" - if log2_rc != 0: - log2 = "RUNNER2_LOGS_UNAVAILABLE" - data["runner_logs"] = { - "gitea-act-runner-1": log1, - "gitea-act-runner-2": log2, - } + inspect_lines = [] + runner_logs = {} + if not runner_containers: + inspect_lines.append("NO_RUNNER_CONTAINERS_FOUND") + for container in runner_containers: + rc, inspect_text = ssh( + host, + f"docker inspect {container} --format 'container={container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} image={{{{.Config.Image}}}}'", + ) + if rc != 0: + inspect_text = f"{container}_INSPECT_UNAVAILABLE" + inspect_lines.append(inspect_text) + + log_rc, log_text = ssh(host, f"docker logs --tail=40 {container}") + if log_rc != 0: + log_text = f"{container}_LOGS_UNAVAILABLE" + runner_logs[container] = log_text + + data["runner_inspect"] = "\n".join(inspect_lines).strip() + data["runner_logs"] = runner_logs results[host] = data print("Collected") @@ -134,8 +143,10 @@ with open(report_path, "w", encoding="utf-8") as handle: write_section(handle, "7. MEMORY STATUS", data.get("memory_status", "")) write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", "")) write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", "")) - write_section(handle, "10. RUNNER LOGS (1)", data.get("runner_logs", {}).get("gitea-act-runner-1", "")) - write_section(handle, "11. RUNNER LOGS (2)", data.get("runner_logs", {}).get("gitea-act-runner-2", "")) + for index, container in enumerate(data.get("runner_containers", []), start=10): + write_section(handle, f"{index}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, "")) + if not data.get("runner_containers"): + write_section(handle, "10. RUNNER LOGS", "NO_RUNNER_CONTAINERS_FOUND") with open(summary_path, "w", encoding="utf-8") as handle: handle.write("# Infrastructure Discovery Summary\n\n") @@ -144,12 +155,7 @@ with open(summary_path, "w", encoding="utf-8") as handle: handle.write("|------|------|---------|-----------------|\n") for host, data in results.items(): - visible = 0 - ps = data.get("docker_ps", "") - if "gitea-act-runner-1" in ps: - visible += 1 - if "gitea-act-runner-2" in ps: - visible += 1 + visible = len(data.get("runner_containers", [])) handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n") print("\nDiscovery complete") -- 2.49.1 From 0ac6108fc7828a1fcb7389a3584783f7676ea6db Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 08:27:13 -0400 Subject: [PATCH 15/80] Use full runner pool for source test lanes --- .gitea/workflows/cicd.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 50243a4..4cba822 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -39,7 +39,7 @@ concurrency: jobs: publish-base: name: Build and Publish CICD Base Image - runs-on: ubuntu-act + runs-on: ubuntu-act-8gb timeout-minutes: 35 outputs: base_hash: ${{ steps.base-state.outputs.base_hash }} @@ -354,9 +354,8 @@ jobs: precommit-tests: name: Pre-commit Checks - # Pin one source-check lane to the 8GB pool so source checks can fan out - # across both runner pools instead of serializing behind a single label. - runs-on: ubuntu-act-8gb + # Source-level checks should be able to use the full ubuntu-act pool. + runs-on: ubuntu-act timeout-minutes: 30 needs: build_cicd steps: @@ -401,9 +400,8 @@ jobs: frontend-tests: name: Frontend Tests - # Pin one source-check lane to the 8GB pool so source checks can fan out - # across both runner pools instead of serializing behind a single label. - runs-on: ubuntu-act-8gb + # Source-level checks should be able to use the full ubuntu-act pool. + runs-on: ubuntu-act timeout-minutes: 25 needs: build_cicd steps: -- 2.49.1 From f1651affd0fe89791235ad580a6526f0c519f431 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 09:38:48 -0400 Subject: [PATCH 16/80] Retry runtime image pushes on transient registry resets --- .gitea/workflows/cicd.yaml | 90 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 4cba822..b2e8ff8 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -581,13 +581,54 @@ jobs: HEAD_SHA: ${{ steps.meta.outputs.head_sha }} run: | set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + echo "${op_name} failed. Probing registry endpoint before retry..." + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" - docker push "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null + + retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" + + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend digest after push/pull retries" + exit 1 + fi + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" @@ -599,13 +640,54 @@ jobs: HEAD_SHA: ${{ steps.meta.outputs.head_sha }} run: | set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + echo "${op_name} failed. Probing registry endpoint before retry..." + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" - docker push "${DEPLOYABLE_FRONTEND_TAG_REF}" - docker pull "${DEPLOYABLE_FRONTEND_TAG_REF}" >/dev/null + + retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" + + if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend digest after push/pull retries" + exit 1 + fi + echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" -- 2.49.1 From b7b66c9e26982e8cce9202eb856b2286c72d2192 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 10:59:29 -0400 Subject: [PATCH 17/80] Harden runtime health probes in integration and e2e jobs --- .gitea/workflows/cicd.yaml | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index b2e8ff8..406cf5e 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -768,7 +768,9 @@ jobs: output_file="$2" probe_tmp="$(mktemp)" - if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + run_probe_with_python() { + python_bin="$1" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend "${python_bin}" -c "$(printf '%s\n' \ 'import sys' \ 'import urllib.error' \ 'import urllib.request' \ @@ -787,9 +789,21 @@ jobs: ' print("000")' \ ' print("")' )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null - then + ' print("")')" "${endpoint_path}" + } + + probe_err="$(mktemp)" + + if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then + : + else : >"${output_file}" + { + echo '{"status":"probe_exec_failed"}' + sed -n '1,5p' "${probe_err}" || true + } >"${output_file}" rm -f "${probe_tmp}" + rm -f "${probe_err}" echo "000" return 0 fi @@ -797,6 +811,7 @@ jobs: http_code="$(head -n 1 "${probe_tmp}")" tail -n +2 "${probe_tmp}" >"${output_file}" rm -f "${probe_tmp}" + rm -f "${probe_err}" echo "${http_code}" } @@ -1012,7 +1027,9 @@ jobs: output_file="$3" probe_tmp="$(mktemp)" - if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" python -c "$(printf '%s\n' \ + run_probe_with_python() { + python_bin="$1" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" "${python_bin}" -c "$(printf '%s\n' \ 'import sys' \ 'import urllib.error' \ 'import urllib.request' \ @@ -1029,9 +1046,21 @@ jobs: ' print(body)' \ 'except Exception:' \ ' print(\"000\")' \ - ' print(\"\")')" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then + ' print("")')" "${endpoint_path}" + } + + probe_err="$(mktemp)" + + if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then + : + else : >"${output_file}" + { + echo '{"status":"probe_exec_failed"}' + sed -n '1,5p' "${probe_err}" || true + } >"${output_file}" rm -f "${probe_tmp}" + rm -f "${probe_err}" echo "000" return 0 fi @@ -1039,6 +1068,7 @@ jobs: http_code="$(head -n 1 "${probe_tmp}")" tail -n +2 "${probe_tmp}" >"${output_file}" rm -f "${probe_tmp}" + rm -f "${probe_err}" echo "${http_code}" } -- 2.49.1 From 1e9e3f0c413498a11f88b5848ccadccb4f07ac07 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 12:45:48 -0400 Subject: [PATCH 18/80] Add runtime-images postmortem job for setup-stage failures --- .gitea/workflows/cicd.yaml | 111 +++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 406cf5e..0f15fe7 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -693,6 +693,117 @@ jobs: - *failure_diagnostics_step + runtime-images-postmortem: + name: Runtime Images Failure Postmortem + # Run on the broader runner pool so we can still capture diagnostics when + # runtime_images fails during platform "Set up job" before any step executes. + runs-on: ubuntu-act + needs: runtime_images + if: always() && needs.runtime_images.result == 'failure' + timeout-minutes: 10 + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Collect runtime_images postmortem context + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + + echo "=== Postmortem Context ===" + echo "workflow=${GITHUB_WORKFLOW:-unknown}" + echo "run_id=${GITHUB_RUN_ID:-unknown}" + echo "run_number=${GITHUB_RUN_NUMBER:-unknown}" + echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" + echo "repository=${GITHUB_REPOSITORY:-unknown}" + echo "server_url=${GITHUB_SERVER_URL:-unknown}" + echo "job_result_runtime_images=${{ needs.runtime_images.result }}" + + echo "=== Local Runner Telemetry (postmortem job host) ===" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + + echo "=== Registry Reachability Probe ===" + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + + if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "No GITHUB_TOKEN available for API diagnostics" + exit 0 + fi + + API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}" + JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200" + + echo "=== Actions API Job Summary ===" + echo "jobs_url=${JOBS_URL}" + if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then + echo "Could not fetch job metadata from Actions API" + exit 0 + fi + + if command -v python3 >/dev/null 2>&1; then + python3 -c "$(printf '%s\n' \ + 'import json' \ + 'from pathlib import Path' \ + '' \ + 'path = Path("/tmp/actions-jobs.json")' \ + 'payload = json.loads(path.read_text())' \ + 'jobs = payload.get("jobs") or payload.get("data") or []' \ + '' \ + 'print("job_count=", len(jobs))' \ + '' \ + 'def _g(job, *keys):' \ + ' for key in keys:' \ + ' if key in job and job[key] not in (None, ""):' \ + ' return job[key]' \ + ' return "unknown"' \ + '' \ + 'for job in jobs:' \ + ' name = _g(job, "name", "job_name")' \ + ' status = _g(job, "status")' \ + ' conclusion = _g(job, "conclusion", "result")' \ + ' runner = _g(job, "runner_name", "runner")' \ + ' started = _g(job, "started_at", "start_time")' \ + ' completed = _g(job, "completed_at", "end_time")' \ + ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \ + '' \ + 'target = None' \ + 'for job in jobs:' \ + ' name = str(_g(job, "name", "job_name"))' \ + ' if "Build and Publish Runtime Images" in name:' \ + ' target = job' \ + ' break' \ + '' \ + 'if target is None:' \ + ' print("runtime_images_job=not_found")' \ + 'else:' \ + ' print("=== runtime_images job detail ===")' \ + ' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \ + ' if key in target:' \ + ' print(f"{key}={target[key]}")')" + else + echo "python3 unavailable; emitting raw jobs payload head" + sed -n '1,120p' /tmp/actions-jobs.json || true + fi + + exit 0 + integration-tests: name: Runtime Black-Box Integration Tests # Pin integration tests to high-memory worker to reduce setup-stage runner churn. -- 2.49.1 From 9301466f9eaeda59a3680da945438447f4cf6c30 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 14:36:46 -0400 Subject: [PATCH 19/80] Add source-lane postmortem job for setup-stage failures --- .gitea/workflows/cicd.yaml | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 0f15fe7..d65abbc 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -804,6 +804,113 @@ jobs: exit 0 + source-lanes-postmortem: + name: Source Lanes Failure Postmortem + # Capture follow-up diagnostics when any source lane dies during platform + # setup before step logging is available. + runs-on: ubuntu-act-8gb + needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] + if: always() && (needs['backend-tests'].result == 'failure' || needs['precommit-tests'].result == 'failure' || needs['frontend-tests'].result == 'failure' || needs.xdoctest.result == 'failure') + timeout-minutes: 10 + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Collect source lane postmortem context + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + + echo "=== Postmortem Context ===" + echo "workflow=${GITHUB_WORKFLOW:-unknown}" + echo "run_id=${GITHUB_RUN_ID:-unknown}" + echo "run_number=${GITHUB_RUN_NUMBER:-unknown}" + echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" + echo "repository=${GITHUB_REPOSITORY:-unknown}" + echo "server_url=${GITHUB_SERVER_URL:-unknown}" + echo "backend_tests_result=${{ needs['backend-tests'].result }}" + echo "precommit_tests_result=${{ needs['precommit-tests'].result }}" + echo "frontend_tests_result=${{ needs['frontend-tests'].result }}" + echo "xdoctest_result=${{ needs.xdoctest.result }}" + + echo "=== Local Runner Telemetry (postmortem job host) ===" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + + echo "=== Registry Reachability Probe ===" + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + + if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "No GITHUB_TOKEN available for API diagnostics" + exit 0 + fi + + API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}" + JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200" + + echo "=== Actions API Job Summary ===" + echo "jobs_url=${JOBS_URL}" + if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then + echo "Could not fetch job metadata from Actions API" + exit 0 + fi + + if command -v python3 >/dev/null 2>&1; then + python3 -c "$(printf '%s\n' \ + 'import json' \ + 'from pathlib import Path' \ + '' \ + 'path = Path("/tmp/actions-jobs.json")' \ + 'payload = json.loads(path.read_text())' \ + 'jobs = payload.get("jobs") or payload.get("data") or []' \ + '' \ + 'print("job_count=", len(jobs))' \ + '' \ + 'def _g(job, *keys):' \ + ' for key in keys:' \ + ' if key in job and job[key] not in (None, ""):' \ + ' return job[key]' \ + ' return "unknown"' \ + '' \ + 'targets = {' \ + ' "Backend Tests",' \ + ' "Pre-commit Checks",' \ + ' "Frontend Tests",' \ + ' "Backend Doctests",' \ + '}' \ + '' \ + 'for job in jobs:' \ + ' name = str(_g(job, "name", "job_name"))' \ + ' if name in targets:' \ + ' status = _g(job, "status")' \ + ' conclusion = _g(job, "conclusion", "result")' \ + ' runner = _g(job, "runner_name", "runner")' \ + ' started = _g(job, "started_at", "start_time")' \ + ' completed = _g(job, "completed_at", "end_time")' \ + ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")')" + else + echo "python3 unavailable; emitting raw jobs payload head" + sed -n '1,120p' /tmp/actions-jobs.json || true + fi + + exit 0 + integration-tests: name: Runtime Black-Box Integration Tests # Pin integration tests to high-memory worker to reduce setup-stage runner churn. -- 2.49.1 From f0b259e1c30d8905ab843cd9736d5926c308d6cf Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 18:01:48 -0400 Subject: [PATCH 20/80] Add tester image lanes and production gates to CICD workflow --- .gitea/workflows/cicd.yaml | 301 +++++++++++++++++++++++++++++----- Dockerfile.e2e-tester | 7 + Dockerfile.integration-tester | 9 + 3 files changed, 273 insertions(+), 44 deletions(-) create mode 100644 Dockerfile.e2e-tester create mode 100644 Dockerfile.integration-tester diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index d65abbc..a478d6c 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -275,7 +275,6 @@ jobs: backend-tests: name: Backend Tests - # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act timeout-minutes: 25 needs: build_cicd @@ -485,10 +484,221 @@ jobs: - *failure_diagnostics_step + cicd-tests-complete: + name: CICD Tests Complete + runs-on: ubuntu-act + needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] + steps: + - name: Confirm all source lanes passed + run: | + echo "✅ Source checks complete" + + build-integration-tester-image: + name: Build Integration Tester Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} + integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for integration tester image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.integration-tester + else + git checkout HEAD -- Dockerfile.integration-tester + fi + + - name: Build and push integration tester image + id: integration_tester_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration" + INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.integration-tester -t plex-playlist-integration:"${HEAD_SHA}" . + docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}" + + retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4 + retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3 + + INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then + echo "❌ Unable to resolve integration tester digest" + exit 1 + fi + + echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-e2e-tester-image: + name: Build E2E Tester Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} + e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for E2E tester image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.e2e-tester + else + git checkout HEAD -- Dockerfile.e2e-tester + fi + + - name: Build and push E2E tester image + id: e2e_tester_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" + E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.e2e-tester -t plex-playlist-e2e:"${HEAD_SHA}" . + docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}" + + retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4 + retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3 + + E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then + echo "❌ Unable to resolve E2E tester digest" + exit 1 + fi + + echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + runtime_images: name: Build and Publish Runtime Images runs-on: ubuntu-act-8gb - needs: [build_cicd, precommit-tests, backend-tests, frontend-tests, xdoctest] + needs: [build_cicd, cicd-tests-complete] timeout-minutes: 45 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} @@ -693,6 +903,15 @@ jobs: - *failure_diagnostics_step + production-images-complete: + name: Production Images Complete + runs-on: ubuntu-act + needs: runtime_images + steps: + - name: Confirm deployable runtime images are published + run: | + echo "✅ Deployable backend/frontend runtime images are available" + runtime-images-postmortem: name: Runtime Images Failure Postmortem # Run on the broader runner pool so we can still capture diagnostics when @@ -916,7 +1135,7 @@ jobs: # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 - needs: runtime_images + needs: [runtime_images, production-images-complete, build-integration-tester-image] steps: - name: Identify runner run: | @@ -936,6 +1155,7 @@ jobs: HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} run: | set -euo pipefail set -o pipefail @@ -986,50 +1206,37 @@ jobs: output_file="$2" probe_tmp="$(mktemp)" - run_probe_with_python() { - python_bin="$1" - docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend "${python_bin}" -c "$(printf '%s\n' \ - 'import sys' \ - 'import urllib.error' \ - 'import urllib.request' \ - 'endpoint_path = sys.argv[1]' \ - 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ - 'try:' \ - ' with urllib.request.urlopen(url, timeout=2) as response:' \ - ' body = response.read().decode("utf-8", errors="replace")' \ - ' print(response.status)' \ - ' print(body)' \ - 'except urllib.error.HTTPError as err:' \ - ' body = err.read().decode("utf-8", errors="replace")' \ - ' print(err.code)' \ - ' print(body)' \ - 'except Exception:' \ - ' print("000")' \ - ' print("")' - )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null - ' print("")')" "${endpoint_path}" - } - - probe_err="$(mktemp)" - - if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then - : - else + if ! docker run --rm \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + "${INTEGRATION_TESTER_DIGEST_REF}" \ + python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://backend:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")')" \ + "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then : >"${output_file}" - { - echo '{"status":"probe_exec_failed"}' - sed -n '1,5p' "${probe_err}" || true - } >"${output_file}" - rm -f "${probe_tmp}" - rm -f "${probe_err}" echo "000" + rm -f "${probe_tmp}" return 0 fi http_code="$(head -n 1 "${probe_tmp}")" tail -n +2 "${probe_tmp}" >"${output_file}" rm -f "${probe_tmp}" - rm -f "${probe_err}" echo "${http_code}" } @@ -1163,7 +1370,7 @@ jobs: # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act-8gb timeout-minutes: 45 - needs: runtime_images + needs: [runtime_images, production-images-complete, build-integration-tester-image, build-e2e-tester-image] steps: - name: Identify runner run: | @@ -1186,6 +1393,8 @@ jobs: DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_frontend_tag_ref }} DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_frontend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} + E2E_TESTER_DIGEST_REF: ${{ needs.build-e2e-tester-image.outputs.e2e_tester_digest_ref }} run: | set -euo pipefail set -o pipefail @@ -1295,7 +1504,10 @@ jobs: output_file="$2" probe_tmp="$(mktemp)" - if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + if ! docker run --rm \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + "${INTEGRATION_TESTER_DIGEST_REF}" \ + python -c "$(printf '%s\n' \ 'import sys' \ 'import urllib.error' \ 'import urllib.request' \ @@ -1311,7 +1523,8 @@ jobs: ' print(body)' \ 'except Exception:' \ ' print(\"000\")' \ - ' print(\"\")')" "${target_url}" >"${probe_tmp}" 2>/dev/null; then + ' print("")')" \ + "${target_url}" >"${probe_tmp}" 2>/dev/null; then : >"${output_file}" rm -f "${probe_tmp}" echo "000" @@ -1488,8 +1701,8 @@ jobs: -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \ -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ - mcr.microsoft/playwright:v1.56.1-jammy \ - bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \ + "${E2E_TESTER_DIGEST_REF}" \ + bash -lc "cd /workspace/frontend && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \ 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} diff --git a/Dockerfile.e2e-tester b/Dockerfile.e2e-tester new file mode 100644 index 0000000..57fff55 --- /dev/null +++ b/Dockerfile.e2e-tester @@ -0,0 +1,7 @@ +FROM mcr.microsoft/playwright:v1.56.1-jammy + +WORKDIR /workspace/frontend + +RUN corepack enable + +CMD ["bash"] diff --git a/Dockerfile.integration-tester b/Dockerfile.integration-tester new file mode 100644 index 0000000..989ef9d --- /dev/null +++ b/Dockerfile.integration-tester @@ -0,0 +1,9 @@ +FROM python:3.14-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +CMD ["bash"] -- 2.49.1 From d4aa16e18a5f30a7a6e7a64f1bf301be82bd0c81 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 18:15:33 -0400 Subject: [PATCH 21/80] Separate backend/frontend base-main lanes and add postmortem coverage --- .gitea/workflows/cicd.yaml | 510 +++++++++++++++++++++++++++++++++++-- Dockerfile.frontend | 4 +- 2 files changed, 494 insertions(+), 20 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index a478d6c..c5266c7 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -273,6 +273,34 @@ jobs: - *failure_diagnostics_step + build-cicd-postmortem: + name: Build CICD Image Failure Postmortem + runs-on: ubuntu-act + needs: build_cicd + if: always() && needs.build_cicd.result == 'failure' + timeout-minutes: 10 + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Build CICD postmortem diagnostics + run: | + echo "=== Build CICD Postmortem ===" + echo "build_cicd_result=${{ needs.build_cicd.result }}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + backend-tests: name: Backend Tests runs-on: ubuntu-act @@ -488,11 +516,170 @@ jobs: name: CICD Tests Complete runs-on: ubuntu-act needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] + if: always() steps: - name: Confirm all source lanes passed run: | + set -euo pipefail + backend_status="${{ needs.backend-tests.result }}" + precommit_status="${{ needs.precommit-tests.result }}" + frontend_status="${{ needs.frontend-tests.result }}" + xdoctest_status="${{ needs.xdoctest.result }}" + + echo "backend-tests=${backend_status}" + echo "precommit-tests=${precommit_status}" + echo "frontend-tests=${frontend_status}" + echo "xdoctest=${xdoctest_status}" + + if [ "${backend_status}" != "success" ] || [ "${precommit_status}" != "success" ] || [ "${frontend_status}" != "success" ] || [ "${xdoctest_status}" != "success" ]; then + echo "❌ One or more CICD source lanes failed" + exit 1 + fi + echo "✅ Source checks complete" + - *failure_diagnostics_step + + build-backend-base-image: + name: Build Backend Base Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }} + backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for backend base image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock + else + git checkout HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock + fi + + - name: Build and push backend base image + id: backend_base_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base" + BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.backend --target dependencies -t plex-playlist-backend-base:"${HEAD_SHA}" . + docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}" + docker push "${BACKEND_BASE_TAG_REF}" + docker pull "${BACKEND_BASE_TAG_REF}" >/dev/null + + BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${BACKEND_BASE_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend base image digest" + exit 1 + fi + + echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-frontend-base-image: + name: Build Frontend Base Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} + frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for frontend base image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml + else + git checkout HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml + fi + + - name: Build and push frontend base image + id: frontend_base_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base" + FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.frontend --target deps -t plex-playlist-frontend-base:"${HEAD_SHA}" . + docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}" + docker push "${FRONTEND_BASE_TAG_REF}" + docker pull "${FRONTEND_BASE_TAG_REF}" >/dev/null + + FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${FRONTEND_BASE_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend base image digest" + exit 1 + fi + + echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + build-integration-tester-image: name: Build Integration Tester Image runs-on: ubuntu-act-8gb @@ -696,9 +883,10 @@ jobs: - *failure_diagnostics_step runtime_images: - name: Build and Publish Runtime Images + name: Build and Publish Runtime Images (Legacy Disabled) runs-on: ubuntu-act-8gb - needs: [build_cicd, cicd-tests-complete] + if: false + needs: [build_cicd, cicd-tests-complete, build-backend-base-image, build-frontend-base-image] timeout-minutes: 45 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} @@ -903,22 +1091,264 @@ jobs: - *failure_diagnostics_step + build-backend-main-image: + name: Build Backend Main Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete, build-backend-base-image] + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for backend main image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- \ + .dockerignore \ + Dockerfile.backend \ + backend \ + scripts/verify-deployable-image-purity.sh + else + git checkout HEAD -- \ + .dockerignore \ + Dockerfile.backend \ + backend \ + scripts/verify-deployable-image-purity.sh + fi + + - name: Build and push backend main image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + + retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend main digest" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-frontend-main-image: + name: Build Frontend Main Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete, build-frontend-base-image] + outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} + deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for frontend main image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- \ + .dockerignore \ + Dockerfile.frontend \ + frontend \ + scripts/verify-deployable-image-purity.sh + else + git checkout HEAD -- \ + .dockerignore \ + Dockerfile.frontend \ + frontend \ + scripts/verify-deployable-image-purity.sh + fi + + - name: Build and push frontend main image + id: deployable_frontend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" + + retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + + DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend main digest" + exit 1 + fi + + echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + production-images-complete: name: Production Images Complete - runs-on: ubuntu-act - needs: runtime_images + runs-on: ubuntu-act-8gb + needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + if: always() + timeout-minutes: 10 steps: - - name: Confirm deployable runtime images are published + - name: Confirm required production/tester images are complete run: | - echo "✅ Deployable backend/frontend runtime images are available" + set -euo pipefail + backend_main_status="${{ needs.build-backend-main-image.result }}" + frontend_main_status="${{ needs.build-frontend-main-image.result }}" + integration_tester_status="${{ needs.build-integration-tester-image.result }}" + e2e_tester_status="${{ needs.build-e2e-tester-image.result }}" + + echo "build-backend-main-image=${backend_main_status}" + echo "build-frontend-main-image=${frontend_main_status}" + echo "build-integration-tester-image=${integration_tester_status}" + echo "build-e2e-tester-image=${e2e_tester_status}" + + if [ "${backend_main_status}" != "success" ] || [ "${frontend_main_status}" != "success" ] || [ "${integration_tester_status}" != "success" ] || [ "${e2e_tester_status}" != "success" ]; then + echo "❌ Production Images Complete gate failed" + exit 1 + fi + + echo "✅ Production, integration tester, and E2E tester images are complete" + + - *failure_diagnostics_step runtime-images-postmortem: name: Runtime Images Failure Postmortem # Run on the broader runner pool so we can still capture diagnostics when # runtime_images fails during platform "Set up job" before any step executes. - runs-on: ubuntu-act - needs: runtime_images - if: always() && needs.runtime_images.result == 'failure' + runs-on: ubuntu-act-8gb + needs: [runtime_images, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + if: always() && (needs.runtime_images.result == 'failure' || needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') timeout-minutes: 10 steps: - name: Identify runner @@ -1135,7 +1565,7 @@ jobs: # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 - needs: [runtime_images, production-images-complete, build-integration-tester-image] + needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image] steps: - name: Identify runner run: | @@ -1152,9 +1582,9 @@ jobs: - *configure_registry_host_step - name: Run runtime black-box integration checks via compose env: - HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_digest_ref }} INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} run: | set -euo pipefail @@ -1365,12 +1795,33 @@ jobs: - *failure_diagnostics_step + integration-tests-postmortem: + name: Integration Tests Failure Postmortem + runs-on: ubuntu-act + needs: integration-tests + if: always() && needs.integration-tests.result == 'failure' + timeout-minutes: 10 + steps: + - name: Integration postmortem diagnostics + run: | + echo "=== Integration Tests Postmortem ===" + echo "integration_tests_result=${{ needs.integration-tests.result }}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + e2e-tests: name: End-to-End Tests # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act-8gb timeout-minutes: 45 - needs: [runtime_images, production-images-complete, build-integration-tester-image, build-e2e-tester-image] + needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] steps: - name: Identify runner run: | @@ -1387,12 +1838,12 @@ jobs: - *configure_registry_host_step - name: Run E2E tests against runtime services via compose env: - HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} - DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_frontend_tag_ref }} - DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_frontend_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_digest_ref }} + DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.build-frontend-main-image.outputs.deployable_frontend_tag_ref }} + DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.build-frontend-main-image.outputs.deployable_frontend_digest_ref }} INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} E2E_TESTER_DIGEST_REF: ${{ needs.build-e2e-tester-image.outputs.e2e_tester_digest_ref }} run: | @@ -1720,3 +2171,24 @@ jobs: find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true - *failure_diagnostics_step + + e2e-tests-postmortem: + name: E2E Tests Failure Postmortem + runs-on: ubuntu-act + needs: e2e-tests + if: always() && needs.e2e-tests.result == 'failure' + timeout-minutes: 10 + steps: + - name: E2E postmortem diagnostics + run: | + echo "=== E2E Tests Postmortem ===" + echo "e2e_tests_result=${{ needs.e2e-tests.result }}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + if command -v docker >/dev/null 2>&1; then + docker version || true + docker info || true + fi + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true diff --git a/Dockerfile.frontend b/Dockerfile.frontend index b66cb41..662b7ae 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -1,11 +1,13 @@ # Frontend Dockerfile for Vue/Vite TypeScript project -FROM node:24-alpine AS build +FROM node:24-alpine AS deps WORKDIR /app COPY frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml ./ RUN corepack enable && yarn install --immutable +FROM deps AS build + COPY frontend/ . RUN yarn build -- 2.49.1 From bd174d8186da23fc631cf4e5c3a9771cff33c264 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 19:22:29 -0400 Subject: [PATCH 22/80] Harden E2E tester build against transient MCR DNS failures --- .gitea/workflows/cicd.yaml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index c5266c7..2b470c5 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -831,6 +831,40 @@ jobs: run: | set -euo pipefail + PLAYWRIGHT_BASE_IMAGE="mcr.microsoft/playwright:v1.56.1-jammy" + + retry_pull_with_dns_probe() { + image_ref="$1" + attempts="${2:-6}" + backoff="${3:-5}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "base pull attempt ${attempt}/${attempts} for ${image_ref}" + + if getent hosts mcr.microsoft >/dev/null 2>&1; then + echo "mcr.microsoft DNS resolution: ok" + else + echo "mcr.microsoft DNS resolution: failed" + fi + + timeout 10 curl -fsSIL "https://mcr.microsoft/v2/" >/dev/null || true + + if docker pull "${image_ref}"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying base image pull in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -865,7 +899,8 @@ jobs: E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" - docker build -f Dockerfile.e2e-tester -t plex-playlist-e2e:"${HEAD_SHA}" . + retry_pull_with_dns_probe "${PLAYWRIGHT_BASE_IMAGE}" 6 5 + DOCKER_BUILDKIT=0 docker build --pull=false -f Dockerfile.e2e-tester -t plex-playlist-e2e:"${HEAD_SHA}" . docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}" retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4 -- 2.49.1 From e246e6d5bcc5eafef0e91944df5a9b1ff660083b Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 19:24:30 -0400 Subject: [PATCH 23/80] Adding the workflow spreadsheet Signed-off-by: copilotcoder --- cicd-workflow.ods | Bin 0 -> 19937 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cicd-workflow.ods diff --git a/cicd-workflow.ods b/cicd-workflow.ods new file mode 100644 index 0000000000000000000000000000000000000000..ff06fef0b81c0ea95ddc002a3c1608908910e1a6 GIT binary patch literal 19937 zcmb4q1yEeg*6j=f4DJMi4M76I-Ggf&IE3KtG6eS-oDc{W2=49y65QP_NC+^vyZht2 zfBpZfS9RZg^}6a*^*L3$d(ZCEd#zskjH)6CnFs(t0|2g)xAa5o1wXR^0Dyn4=SKhs z8wU$lFQ|nH6l!l{X5wn&q zRsBE4d>-b18pU%@%E{5v#>(x}zeaQ6VmC1}v#_^#zIFP<{(rZG{2NOf2NNp`7j{V- zR|gZQ%YQH?`!~j}PEPhv6GsdC|6oV@Z|p4XolIOU{*$Z!ZZ;QJ6IVAEQ_wQ!= z$IJh0_3uIo{p4iz$-?FTG2g%Q6Y2zYgFdgP+y7@&{xSLAexjkF{p)pl4&VPYjOU(< ztCzin3!A5d{i5EU%R)W=(^8vcu9EP9l1*G3B+hooCF%T+U3_i)`k$}-9O&vr5`iG` z#PqhuW4ew)>z+^hOc`{Yho&TV*CrCuS+TTr$u#ILN`GCyPtu5G3t#@KAgYjK& zLQBswe|q;Myjrj=#ZiD=yR~iMK-v9_P*US*h{0%51m~#1sX>FRuRhY}1+t6Sg;_h? zBMfmE^{sySxJ-japDn31M|bAC`CE;!R>UzPT?bI}mnEkNVU|?cuo%J+Y-yHHU&)*b zY~A@d)%ipnzFUcV#M&0UspIhaGFntpzE~N<8fPzE!ziFqn@@Y{ZI_{BbGUhk{1@`S znvLYsjk)1cx~OEi_dmtP<8)C|0v8c+o$W~?(ex8p^!V*+f?hqk_HVS6oY$qWHae3b z8Nn7pih{9Ay#_q2d#tY(L}u!89>7PK^09QlvTp6Vm-=nR>Xhk8ar56czrFPd@si?J z=q>ynlln~vj`jeJX7%e?mxY$G4KR<7@bVO#c{X2*xLeqCfcHht-msu;Sa2LpPGr|B z`Y^rv0}v8o{M_(DZKr{|iudX3G6ebmW`YQhBx<}O0y z>L~x0ZOV#hCtRD^`dL_Sw4-Y?M&*T4cYGUv3R_8bS0uyGyg9#c=o#4x53WQ!vi&xE zzf4=rxDh*G6%)xSyB=sUg6ChK@(3gfe#q}A1Q*8tnT}glG+~>06%&4dXT}?ibRut0 zb8&Q;Ii zmp#etQ28TYh*3l-ydiVw&s#DLJ`EmsOaVBpJJ8o!5jW9PKS}KVRFizSvX?zRxOSC zXgKq;8dKF5-Kf{P?-MWG%-+c6(>FiNX|r$=b=@tkTNkm<3u7}DlTY(x(>>g)(TmHO z_XX?q5X)GUja=z=ON(+V50uw}34sAln}u6e2v?!l)YNvmFN%Xb_Q}!_pU0GyHew}z z?FgQ%d(d|BQNI>ep<@X8+mxh07#gX%{+{HaBj1kO;*8#1i=DPwK3wv;WUxNl_1gg2 z&$KF}etj8Ex)frBDQJJchwtmP_D0?hUCmc7W2(lou|5(lR!=RL`qM9`*!}@I=?YB>aG|Im^8C&Y`i!&6T9$QPA z{KrYc6TZM$-f_j?(rDN8T74~Y#%h?uZ_$%?*$`~W_$gQ*Ti0@gcxn&$x%QKlc|NQ; z(XhM#SHp5S`b}@`k3Zx-1CKokaVet}Q zi1&luOqnd6Zhj!0q6r0f_Az4zBVYJ8AQP9mQla+-gps}p8=3RY?ZrKr`*HZ-myI_< z0_bcnNa&A$>(DUzh(%%(&&}}8BKYw$899j1T@9I7PWbTh$sO(TC3q4^S#!2A@P@!# z831Yvgs<1!L1D*>mHCvJo-H&aT`CCw9_6w|})WjIdpA1ae=m2n@6wvz*U z7n~i_M#SO7;_;M;;g(P@zl_iEA!o~(hE%5h#}E_Lrt^q#3al0!I*bDBf{2^MY<t8uQ{J2xnZR%>A)`Xz_lfj8SVz@I#;PW4I$UJr54*6O~d&a z^NQY2_w^|+ZmoZ{$lou4Tk10k@2h+TFDrx)=1vk{%QA2LrW9wP8EAhRuU z^YLX(WOul$fDc>ImzI-??4^UyFJVPsll0b4-Ud_&K0+*O6{IpKMRnqP$}X2X=A3db zy?*YjSX+}FOpZ|;v-*@9U+_z=2HEoui6U3z$anod`LuDb?+RN3k~<2zrIc8G3X(Z6 zYcOf>O(XY59?L+V1opW~5Ea+7Ta(6jxphVAY#Y6yKfl>)6-lP-L65loA~1R& z80)tH-wl%&P51G-lc>CFZK^~a%}7~k&n=ATepS3HA}wH9LWL;Gh|hPPHUSSgK@+Ct zTRG%Lz?8fJj&eP_E>*d`<&T8sT`Kp6beD{_XSFWfFWMT&i@Wf&HsUlbx1CeH3!3 zT}yC@-N3;IFwv*leJm1Vs^KflpRDPb>>X$wUK)7AThK4PQw0dg@TdvT8KN(VL)LS} zy2s6|0+><)boIqlR<$5>unb`!=X}T9Ui#WT3vwj6*5?D{N?h&AkHw$Lmn!0n6M)e) zV+5Cmm?O6pA*Q6+SZD&h)Pei9OrJU%ILHb_Z-o?EvsTfuUg;TfA4$cAOyLw zkSM+N*Kj}~h)!V|(UA=Ijm<>F1p*-9?!H6iH3JGXq8xXCXK|-fN3JOK#G2&L+D~jdY!o)bV^wBR zo?h}I6Mu$m)Wq&V+w~Gw0j^J8cSRaeg3IRRp62X@ry) zq5}X_f&jpO@$-L2@35Zfor8&^jirT)E88b?%aQmI#~=>8;3JU`i_tl8b`+FPjDrof zneDuxIweq5;RH-VdclNAFIh>^h`MuQE8^>_$}ZpkJVcnjfPGY zC24c9?n@>(x{7Z3LCtk9E5_@J`fcQ*P;u4|43nSgEnz>YW8vaa=t_gZu0a%y%8EVQ ziDznp9oYw~ea0UQb((Y-3L^S1W7XrR~K>bFFS;wrWoyT5^^Dgs=9>dJ+W)00e>lQ}{5S!)NB? z==#q|>X{xN=sCD7@DV)vw;|=G?(lFISbzL9rtRQSpggI-t}I;bGv$vw@GY1d4GK`L zee@$iL^A4XGWJ#H|1nBKn=@d!^r~vYE3U~rG*#Zr;)!2v=HG@~qJ^uOxMMOYSOHV5%feb{lYtqar`=h%!*FSl%?3xo5dQ-+rG0}Nj#x( z?M}$-m!l|^?OUF$mSoO>xvLtMJ*%!(O^v6`8mBE0kVh|jej(H6YoVRouC8pc3tHPE zeyf;=rs?|BWY6+m#QFKON6oh)OV$GLW9A1)W!KB@X){T3CP^Jz8Kaq!UwLFS zSEBcixB8&1U}=nE!o=#nBDt6+zXs5>;xcY& z@-Tmm;q;FIHFlYKv5uUdZuSBOE-Xr&-gmH#vev?{otu3yU6{6sE`&VmiTrukANc-8 zI;W6DL;*41S{2d+-!=UxRK+Rj!FXEa?6L2d(m8)Oa+{SzZ6WazHAQJ%KvmqS1;Ft( zTcXL;rW$Wd$J{4D;>jE;V%q7DwK?{BD>`Gep=)Ad$9)t#bFe|}DGZ{Fx**8!&9oJL z*B6$%;yy|#_RS^iAwIrbRm)*ZVxVk=Gdf&$rQhO_&4pg{BKB{N9i4=O-PB*MsmW{QCFu_`EG|oO;2zYuK~^X60} z66FnKe+sYY89%Z=+a{l*@CT*lhW+X0ZtKl@FCA4kvDEnOz=ml-W>DjJn5M(ofrzVA zZqRNeMX=V%DlXCvG>~lMD{z)tDw+(*)PVcpHDpg29B+D{xV+X#{qyR^u!!$D09JFX)KV7Lkl}+*hLXl;Fm^JY-R{Q!s?g`3n{F zn(Eyn$oPkZhzBI^r;mr8ztgqWeVF$`>y(I>f~4%vm-aDd84snU*HLTd)6m)|`!XVp za7FYV*;cQ#LpO;#TY~ZQ7>?9?Il0$Ap{^V-hu(#)_-tf|v~5oKp^Q6p!{yi(J;*j2 zgenSUstq4;#}e0X2a8h4i;Z4alEZj z4^$nsC{V~TN7HQ@#kuN`ZhX1Dm~HL9`Y!zP2&4W;?i}L3To!%7Tw|Dg@m_sK@queC zS>tl(R@96)sYP?}tp`^CJnaq!%zSYKnAZ*T@loscIi$HQ^_wx*^Ayp}(t!UI$r5`QWP1LN*9rJ2QMNr28amRXDsg2^Q??eHk8QO%}r`4k6K|J&|e%y6KNYf>{--p zez(^oz|aXT<^$P@dCh}}pbnago3ASTkcy@9?rRvD&5ZOT3C9St**y13{?)_Qaz)}8 znBtvVl`FIEHw|lBw}RuxOstLfuOnQc#PV#pM8+RXdYL~kMv}fh52D#uCs8rJT_?OH z_P7R<+N_z02jxQhPvI`7(d^X8@KA)Z~sK2E)=OBpyq-oRNVJ3hwuXrJ5Bn7`TZZEszQwo3f^oir)> z_6<;9cY?D0`Wnz0jaqEuwU#$ce!l+@YxeibZ_+Bl4Z9%NyuRbdrQ3Z@R-0svDUp-v z7S=DfQeDSgQ0-q#$FIhT`1esmUAh{qJzmq*E<#vwNI{`$)2loRC!KF?EFSg$u&{nc z3j2lmTi54xw4Rbb3Atd#F1N7elby*Z+xPF8%P1MywhA8Sy5Bmgp{jRqeB2!g8FUNI zkGNK;1Q{D5M)c+~A0goI8lKtVr zFEt?lTu(M&SVNRk6r-nfd|Z1+Z2x6O{gzYFg;eAWGd{}SSs(1`s`L{nHftml>lR~{ zREef1>`rzL3D3j7Ft`cpUJ1C)xD6$xu9HS+r+ut|!Ih}pSPA7N?`Rzq5v=dZ4_MX4i=tx7YBuU>!yydXLx8+Tq( zDUEX+)}%(tED9Ih7{+gqFjXfOPQV;W~=Vyi3K}` zfBs1E7V}0&N88RXY-mdRrde0JjQkiF@%C#=6cZaA&6v@$uO3pW?s-yCnmQFy6lVFg zENHk-|5P)CEj`009BIP5-%a~%0heM@uGC9rFQc+yE#)h+D{Wvo+KkBdQb|eAiMfF`>Vtk4V?%x<%e=UH-XsY>$rJs|IXR-zDmqFeg_l z6UtwGRXcOz2|ef1SJNPuVMDl;N%0uw7o!+Vxb*kav}v5*5CmWtli7J1v#XJ5L3sIz z0LwqO8OLJVNNxxGdAA*nuR+(<2+Q_$WbG(18JX=7TZCfNFA9FeIf-Nqr|>3&N1dFE z98OXP{A;i&4_(5l%1_`nCE);*t@4YD5`pnw4h0qZ_E{;OrC5qSvnyjH;y&{cZuNy-grM3fk2U zV*eIf-G(Y4BLp3*g3@|CR8x7QE7E0291%5w>XWw7C!TMYXXM0;2HcSsh;l#x>=jOFnI zPes<4noNZ8ZA$T-IzD+yA2zq%G!=$s2;01fHE${e=O;o42djm3`V=Z)oI;QQ z0t#{*ym9JHQy_)O#wPGlMQ!$dR%*mBn)9>AIAX|+5rZ0(JSMhBeyiLS67==xtrIJJ zr(7g7uJ@lpPrqzlu&ad~0QPBcAgeyMLA<&JCqOIrH#W#RQs-_rGYcA) zXdUgI^F#aIKyprViF%R>H;S!ngI7D6Z#Irzu2wnbPkxPwm&f5~0lWzOWK7q0VIl3t z?`CJFns7LJo1`bg+B9ty*@T%%^H+`Y1|?y*voW*3o$jTWF#h>#w`;xM{G*Z360<|6 zwV2Z$-cWj_)=T<+44xlQ@rh7r3LKuXPrOBy=c>N1we-ky$4phXFlbK`%`ad+k&Q1p zxv{}_h5B5)Msgch6*iRDV%2u1^B}B31)jJs!bNXc?+yGxezZPFb(qUHHD5<5GT$_c z2)X$EP1!;lij~UB@lt${nKy7X9J&0FEyKRGZJH8i=!NQaqHR_~o-`@@>vCvJ($+%k z`_4W1pR9#~{gTla836c1``=*n-&H-i-+izRW&q%y>z^W%rnQ@csiTRFy$ieRe|OoS zj#d$>%5vBkWdBrZuodK`)d2wDa~1^tKzq)EfGKy-Lu2g00n6YP0v3E z24KCtMgrHPmId3VC!^NIZ)r%tv>3pa=tk3(czF5TnpW~e*?Q5_RgxKg6ZjwM(hu`x1F#I_YhT}BN;+*?jfc$dD2fAp=jEMPCkSFnjQ^A6s6WSrg)} znq=+#4H7S$To?~6iqI=NC0)QZ{bY<9aYOo7IX_LOxLd6yUP`CzxYlJBJ%2Erld1+dSxoyP z#bm?>8X#4BW^=Rli;P+G2YmYukArsYVXtn1cVgRM;=S*2i#dsglSj54C%BIm5J~Lo zBP2*v?&{ibn>G6K*ODsA>&oW(q_Plh-{^LW_^`Ihxx>ealNE!20+rBlOyQv~EJ(Ya ztk){7c~Ni`d}D(JLsvhS?AvJLJ)Z}XiA3IW?oaI!SNh-5#kppF2E7a(qFQ3fbhEsF zGdnmaXXeI8?v4=>*CLbU@ma)8r3GbAD|V{7fy#2sQGd%ZI}g(%xj}}~t8}o1=Ie;U zutMbUgN{Dl(kT(L_R$L@!8=keZb_!j>~oc{n=@f#Bnqjp&mC1#Sf9tjVY+16s=&#v z>W&v28zUtr#|>RuFLStsTG^|LG7jFsY}cX}QJAX6?XPGe{b z8}GaBB^8`WE%z%wZ6DyM>Y`+ly_o+(=^QN_p2ZvWPSzxDLf)<`HPg2^t}23yzO(s~ zS)g*z&(T!;_9HQ!#m-OLPeTM$U07l?#t?r)FQc80lVJ;(;LQRpqGA+DxKEmwV5?oa zI^FS?cce3BBW|Ug%Lv;+y7h+H)n!eN_Lu<85Th-_%E07f#&`=A_mlUu{#x{L!5bFazR!Mu`a)$3V0y5jB(sV{n5SO3mun zG}~4&bPUV5uA^Vw699d~LEt!s7OyjW&C!@=JtZua6K!0aH2RC3?`ny0?izXF{!T&T53nAN(Ilh$0+9P&CD#|Tv@BH*zr&T7AIUZds_JcgomhXcAD zFn!-Ba{W?k+<1x9bMp0LL-a}`jb32KD@{wRn_qPRs^fh^q^Xy{#-RaXv~DI2)J}{5 z66W7X#gqE9FizsxVK{l1o|fWrOc%y# zGM9-5+dKXF7WK1aoMa*8Hz&Q zkHC`Q54Y{eBm&OXo-3S)a+T=%aA3U!@rF+=8rm5e&08amlH&Bhy&(*Am2ZMAor!NN zl!NG|$Tvkrrl^a#GHoRabbqnU% zO}CT9cIdfwfvr%aU5KY+5t%iBz&DpLX!pBTNWc5Q`s!#H7U~g1O7F@yS2KLjcsORc zTV#}rkGHooH-sNciWreM7|;5tED4_`cTcB6Us1J%flZP@-aWq5quu)btbs0nLN0hkp3!W%RVo-h%8G*v;d!Ney%x{%Z(b8?(0nuH0X@;-51e)S@Z)Lq@fkk;P-JDvJUWkZ?&)wu$qTNKm9>MJ4l1(t)NH zs(k~qK)+w296m|6;sPI<+`=$}A9Nu_OS>RH|4DGc z*K?pfxK`AdD9!)^Q~dlIzu#ID#Dp9Uc=0nGPkd%hT-uzNB9^X#l;g13;?pV1)vg&Q zR232)B}!c>Dwmy2=5SpiBvF3kL>-z;B-hMfcVv7L56*Q#9CKcI1j5g5P)J78N8 zO0am8oTfzfDw;06HTr`Pb}Qe&vyZM8WE_*=^}bu!RfH%k=ceTrQ0qv3kMGSt6K#7? zI|wra>-9jx8L&td?=GWiIZp(>eU`>H?UIRdN>a{UgN` zc-`4gd}(BcJ^E#?fT2HNXWjkRO z+j%>`zhcKINKWU#sXm{DurBUvo-qxCMvq&-L|Kvf*3&mmb)81u2f&Pfx%7)2YATw8oAvc(zqKw6Ywlgv2#$S83wk%;K-*Jyip|P85X#faSPQdZ_Hs2ZQh+ zRdK*vha(bH@1D4YW`Qllfvp#y{PhL#sj?`ojMPj+#h4u8l^?dD$h7~2owDGVrcX}>XeR_;0#yKqQF zQvmgwTZDtw3=o1-y)!|fRJ%%XsR%cQJHUZK$s&I4EY|Obl{J!aBYz6t zE;hr|0|w>ZAG$0`P)_FbpB6gymFG=INemH3Wb8b#hhJx|Dyq+uR-rX3L9UP1>E=j1 zGmVb5Au;^tK%qh_F$xOy8E~l2^9{qvQUOu_{Rlmqiopb;?p8dC?Ic@p2eeA-2bj;9 z-r`6LSQXNP(SC4h?o9(0^5hZ-eo4v9CcO_9Rb89}pT1QI9~0649G?sHDP?_+Pu z+--}ALEn1bC-l!jHnZ-3njX{M!k&~N#?M*mH7bl;-c`1X{9X;;h2IJg9}{mIU&x3m zmW_*L5=Lg1^_x6arLH(8_Ny>MkK@3xwXq8MlCa)aa+kY3Y}{_Z_1U~bU)tk&@-*92`5 zg)NW?)$MeI`dZB|OU+roajNhZ)}61K1ujo;M3=jcwGs_uYGNV?8Sm%y2F@^~6UCB9zGvFgt8J zUY&dRd)`Deezt35C8iW;EF-+x#6cEhrN8Ikm-TS{D&jI)?PQyyhYri2yHgnerMSVl z=6|3wdag%yxkINlUG2ET~|e`Z+fBB4*aN!q{pIBSz+d+ zYM~yimWdicMnh#=TttT{WUSMJvZ)(R_bvxJKpMRl7M$%(mj6hr-ZA>+9;s%VWaP`w ztWJ;o_^R{a=q>69(`+4zu#oUYeF|A)xA{#habC;!j(NKJE_@&+7*ssudzzXS{xL%_ z;xc3i=0p7y)TI*rFu37~Wr!cMMj#qhAgJmrVeV!Nk%SQuMBAMKn%->vG>kzrS++crI&Ktn0W7jvb zS3k1299yG-h*k7g9GSKqO37fDd_$Z4h3-fC5BSq8w>Et8tAfxy{3h^kCxeZ6k_>;q*@vG>=#|;=e+@uq#niZxyRZ zE#>)d%5p6xB$7OC8k$8Dd%|E`!X=K|t^~`k)OZXU8(;noY)yv6mArAeB8@SmCwH>- zdxx&BRy{K=?Y1mb7CNNpd6Po#`A!!K-;ptv9NJ zB{~midM~NC3zhNxmC+!b%&J2gP ziyc$?WleHFVd2MkTM~Yr5(aoZ&?8C3m?BA1JajwP3||E7a=?VD)Cn zHMwDxt^CBPeuI_mZpJc}zQ7H)$CpWV_&@ei#G~F5R~kEB z{zerlLG-?*R+FxGCmqH)oPpUrzLb-QSC#0@JQ7F1L}N<*9Dc`(LK9x5TRsAT=_>E{ zgw(OtlPl(lK)JJZEz68zGM&I+`%{c5Qt3NFnYQ*_wZ{f`(sAFiu%Z4Vb;&Z>Uru+z ziTk2?xaPJpTS`eu9($m|WV5Wj)tQQE19dU!rO$%tnSaJB32( z%(@PU6;Q9B+?1FFH;%v44dX7K$FRuy9dOD~$-w<14Z{hFQ&~wMN$UF%yf+kdP*eL!A+;zT-4xpR8R=yUK#e3=yX z?Zqv*x*rg9>R-a$1=%-KzoHF@1l)oHxxeavek&thIaViVLgDR{$ga5pPF*RGj-4BE zVLRN&ro??_P~L6f@ly>>tv;E&L$vhA)s6ykq2wgtJ^M;>O(d7KIH|8MOgS+}o$jNW zVC?mLOm1{t}8nCnz_^=KmTSaK#rib8@06=~kl2L{)^Dv%})$B)_8FDn9 ziHqHb_)H@F@;~oO{d13+Pc_zE{9>+Yd&No6k~0=jXutSQ(UhG~xclL}!3fgrqIyDzShP^Z6w@dH^+vW))jtN^R6450bCjkPa{-|E(zR43TwW zKt?DBFV)94xXl`mf2gGeq8{S`HS~;?{8!kDOoNk!FYa9O)QF&mZ*4AhVW}w{6S;DN z;E(VHiRa3}Yr3mF5dS^s&e|Sr7ui}=Byav@0$46|#7$WBh1Yb;e4BB8X^v1ju?$ua zRC}|5_VwE_G7?gg(9`%PYJzJ-{O+P48(|(JwDQn<^l3x(GvstfXW*D1$Y$wcZ|b!d|WB8W?=*sI}Ol#Eqg|G4w24NnEJ{5T`&x?I96-2chBb^AmgP?ux}nM+Msx37*xe}SZy`?6OZ-&(S9 zd2M4y8r!-GR|$@cOEEef3T!$SuDV7O`UcijRxF}?6qs$KoJ;$86#0$FA zN)?GOp;I@A<^}tGpe6j$ z>;^{STY}clKYeE=1;0Oj9uZ8Bg{Y$5+zl5-_&)(2Cp!^&-X6D^tD$?Be@>l7H9xxB z|BGJ0nkT`Vm=)|33Q%7QPN~%!Sr;8h>oq8`%V`_b^{KUEr9c$V@LAb$qbC*L5xFlbEBT}!0l+tC~Ip9c?@LeVm8GWz9JpCGzQ6x~aT?f7}Bx zBnE+;0@R(;Y(2zRM||eR7%LntqAZ`|>TiQkzHGKlgA3wC5#Jc|z`8A7G1-tBULUzM}sMMuBEj(1>rh_xyt=IajVd_dZiYw6(- zN8nWq`29y-O9o+Eq4$dNb;bhD0mEY9(60?YCVjrhlAH}iLibSe!(zgZ*JrfmpOvD2 zc4`qWM>($XzHbgLpuy-_XX2rm2D=IyyLWb_fofC5YmYd`x4(fy&>>77P@5PPWM60A ztFZwaJnZ_}mh4B>$$2vl+Z~foMRV`W9)6^bt9uZm&Qdn+_Xq4K(JJQrcMG-!HNpVA z{v56A4Xj19B_z%D5u3ibZgBzk^vu$P`nK?z;wzHkul^*UFXq99w>|pek&=8U?$7Wy zBxZaLx1I$j65t)@7P~C(f$Ao}yB@ay{!)s#y^VdG?{=T?s<=u8%Gs&k0qKbpWsHRK zUa}e**SAgIfT6DCK8fBUURWjAb0Pz1&Hb}b_5WJw{&>-<-oxr8a}T& zjuMrBhTrk^J}oa52g`92tx-T*0PGxiG`0u&+{PC>|9q~c)I)yG7Ki7!+m7~AOcxDv zr6V}qW=lCn&ne^%?AYr1PSk}<_xD2bl{*QEppbzyvDoXp$GB$=?i^S8*F9);9deA6 z!j~Izm=x99*}gUJEh!E~KWL@agO4W|vJfadmZRSBUjT1rFkz2C(tRnc+Gik$IKj2Y zSHFA{VY0qeXcD?)`%?3}Q`Hd)B(ZvV3!mB|y1iy?*K)Vx>>d<$4nI+cs5X&g39mw| zJ)YWMbi876iih{MJcirsE0_OLp7rSv#xujyF=>JTt(=r|9rLMtXT_e0IY-MBxF0jC z;sG>K_y}jk1K70Z#-iS^0mRmr_AF2ObdONJilP_s{usdYi1Bp)%W~8WyzqN!=f8k2 zVp3P!gHFJYz$u*Tl9FB`8Wi+-R3pqfz{j66{Y+5+N0JzA&u%iTegmyP<46%BpO{gp zE^+^s;_6?6Yu~wVpa>u!?e5g7c4?A!oP=-sNWT-k;y6GHMunq7T3x}Ph1yK`HE>(E zAoUuM`V~n1ITFP=F5oYxK4Uk`h@GBC=h0OSlg%t^MsB_6aN4naRj(NQyrci-Kwfk=IX2S`J# z8NydizAZoVRB=PKBicuJ6Ay?0EX$4_8FUB(KY#)ZATfKT8u(?R#`D!O2nj!g#1kwT zDynth<*j)6NbiSu^#pjqxajHhc|q|j0(Xn#{&PSU6zVBbFYs6Zh?^|)mj7^~c!MIg z1L=3()wi(uVTCK6?3^K!m*yK|gxCEsai5uLsT}t{CV*OS7t;Ty7VM7fYco9$y0Q5! zb@9<(qKr2Te2Qs!1ZkB5f39zbmuCjI+2#hXE@el)F?-E8mo{vCS{AORmLiYr{6 zL8}v?kbbTS2|#AZLo`gFX)iO;=ve9zzFQRluBT|~Qhd6_vGR8NY6jhJ9^r`Z0Akb1;vri1>l z5dG;D_6U)5D`AVlNNd0cRn34!hxijgg4Zcq%7kwG;!bp9*r#S8aZmX!qLI zq9B!iX?bkWWC(?e6Pwr1B2`%TXJvUUlLFsya6%2I@CdQ1w)k#P+jb#YocBy4(_#tc z6rhI9Q(|bSf)yShU(OlmVc`*FNX?EX{tDR7z*X$n$XN8}H(q1%1OR}^_RqHf;B+Uv z-r7SS6jGnGJ>MXsD_mn?@%RkFPs$@p5!*Ad1@kSV)a@Z8^af$StaC=#@y3f$_Er<) z^Mtv`Y4>?9XqEwU%$3l2q;w~ahsFSaS-aBRYKYxsbnwrqJsHwR+%ZDs(Q54c-)J7$ z?bI<~H!a@bGV3DD&M5rh+@{ME#Y%mvq#fU&o9FNofl7Hao}jNnTgchI&N|JtM8jXJ z*YYrg#<%r}BNy@L%8OSMAM>LmN8ww?j}Q5Rd|#w?xwUrOH$4JJHm*p~KABK}+WUpY zvik~d+VJBl2nl9NE2x@4sq6KoUhV;SgcU;<-Yx?kDaP8iA?<$H zBhya_N|vz_tEc$rC@G6yOIW<#Olw<4@p!{+&>30;6qB`%#~ZOe@T93T7PSYWuF*SH zoMIQ0t!unS-XtHsU2ofjkQ$9_>~s#aTsEU`Y;A+?#=W*qcSi}?J)7>;IR&=J`^B~) zp1eu3_9lM2(Tv)Vs{J{d<04V<1qe;GSy#zj_^4YG5CPzzq|vG#DW-2OsJqEEYhm44K;c=_#Qs- zc9Q!wTlTfBLb7znFAZj=@bZJC0)4k{r*QmXu)j$}(|Y7TEdv6L)uQKr155#1cqZXS zubVOvYroN)UR6C-1W*H4#f{zTZ45{^pYwuqusQh@7Hm(krEaC?5&5u0dH++c)$B{L z4l74Et|@~50pz2hF+utu%Ze7h*Y&V66N-JHqN;ttH2i&u5lhFP{K&hr&k_FBw&Um; zygn}m_$5n1)+`50TuP{|%O1*pG3=cS@|5&30!SOO0*zfy34K>zcs||AJOO?O798EW zTPnZ3TrBXl1~)p^+k?*&+RjrNa5USFRN7-(jHZ!3KSMU5=*OtzhrMvo<9AQu7?})| zLnnR6`V>orA?W%oJsojt8DKlO7uwbX`nZ}g5h~qjSxd?NOCH-V?3)FiTli|Sf{Pf5pUv>F6Tb&Tt~h|>;RE)AEQ*hTuNrIrJeZC}L1 zmF4+H>)^;+NGpg>^#Y&B^y!5Du2N;+2ooHfNL9Gk(mIKu$o)eiR()3zy;jr71+ZKd z32q8n1wOS`>mq>se~FxbO~{4RcKa+fzeVjQ%3?(2(zVRSw2u4mOP3VXe-FF+IYEDS zG`gP|3|dW2&A<(+kKY#IWS~K6P&Qx;*3;B_OTB@k?$;Or%*EwUp}fNVJAv~oh+iWF zl}-4sISe|@-Br(2?ZQxz!hzBxIfVG_B@DbsUq)L;9_ye8wOpcjK0*(2%UA6ZcM&jg zY2z0t1}zDmdd?2rAaQZ;_Gksj+!s;t4(D#M=i*Z9&djk@3QHY>YYvRA2arTnEZY|P zok**s#8?&uW()j{Q-pHQclULyvqKY1^fVm#hYt%yyjruBXp4~fS9%x45YGaI`2^q5 zYDd0k*0Ver{S11CP68h+0WdN6{F)(30we4OX+s^ej`*d>AHn39$;*$e+Yyf$=RxI~ zzviC=KNz!~#vK^ACc35k=*sN=U;BAPNxmSKS*Q3!;cVtJaWlHt-2K}4uJHeX{GP7& z-^J&&JJq*}*Z7?1U&6h?{LTMqH469n*z#+N9!l?c@cR?T^B-UT^1e^~-}Ha01h@$N z-*TX^Y5!e?7c~c2nr27ew2$VfoM~5}_G8oECySml?t1>cKCkJ-*F_Jib~@?2_~~@v z^cO~n?yPxy&R-(mu}I&rGC9BHMZ-UDzQx=djqCRASy1^hRPyRkH&$V>Bh#0gz6VaB zF*His&Q@6ZuSbd%UjF&0PZ$7r(!xR3#x;0O1 zce{d_w5WsQClxLm=0@gU2h{&HGIKmg`@-=0)x$5E8$P$BFzhgnDLJslfMX+H&gC}= z4ZDtJ+UKugz2&?>>edPd`OBUn6+%o8fi0cG96vmE9(|?V_GZtl32ztHGvv<*lX%0! zl*Zny&Elbb%a$eh-SL)&JFbn(j~;mb>p<@R1AjT*%ob4UNNj9lP79Op(bl>6CgB3Z zd2>TIul6f)ObaG*$qP3(om+I^d;IaQjPC_&893||8K&IM^?zeC=c2uXcj}9;%-sde zWeG*>F^vt^qB!)ToswBPHf4Sge^;_P{=A#WkAK>J`>ugTAb?9696m&@yw0Byye;49 zMoMttg-MLs6F!;Vb-9*wt0AtD;pY3Ox3)&5T->^$8eiO4b5Cto(w}c?-u0Tlr2jX& zoRnz!;<(aB|0jtDJn*R#O8b8H*Gc}He?O~91AB>|G=Cj<9nQVw&IP3pz_RmyKgWdn zhP0aNiNV!&j3?)OI9+!9R*Qb!-fQ*(t+TcMoLPPTCyRkc*CBy>s@GS&v-u~)wEio< z!p4@5Pak#F>RNqXTea=_jt$#FHD-!A=_`LIUN8TVKVz>{+p#}c{!4oW4*0TasczQz zdi*GFm7J>QkK(i)(|@z?+4tUB_Ll3S&z!$%9efQBWF9{_bz58h1m@6{a;E8#UZuC?x5X*WujW{>lfEB+Do#$R2SA&{;-*2&n5c}#WqH5F;9eFTyuZmvg=;q zgzJVEQl2cTl5CXkYXsWy?arp}Zx1Z`%(`LY%$h%TKIe9OT(0=}`szPM@y{JmX4Zc% zls0|h+?99lHsg`zGMOinub+=xw5cQXp@r+&^NIy7@;}<@6Hacpr~bJqT4>qqD?eG5 zX+NFfkzgh&@p;WOla2SEMgE_>H}T{7pX~o!;|iX0vDlxN|7ZEXA@#TzV|RlqWHKg# zMTcpWi!U@9>z{qKH`!f!g@=91df<9m22WQ%mvv4FO}Mx4iUL*nx`sIFdiuHP!}sz6 zH?6Fl?3j00LBRF9W$V#xMoZomsM1Iy)kip%~`Dj*Rs1~5AXXP|CRlAZusoDg{Bic{MuLTIb5PEJx$qb z{Ul$ZP=3!K?OKK_`_9@boZSDpC}dZdG4uO3O=WU^!scZ~Op@~hwY4sbD_XF2lx1kD z?={@UYN_1gbvv{)`d|cmsbv52*^HCtu5MoBeE3Lg{B{9{=+MiS4u?ZH&TU()Sm<*0 zIM0uo(s@oSOD{7v%!o*OdU^52*gj?3y`Aj3OM3RG9N7Hy?VJrhHWzO?N;#CLtX!hq z_D$(c$??Tz{BMGn#ffE4S)0l6@8!M5E0-5?cAUzuuDkTx`CWZLO?NvjmDD1c05ogsuyrf`I{a z><%2)nAfKn4yd~*9l?@Z0kSrp#Y%$0tny_GzD?a6D$eC^dawt2W_B206icVUg!W1 z5J1<7yw{E!p$*8&=1Q=mFjK^%487fdIv u-4+P@u1VwGdjT^A7Es8g2dH31fRFAtis=AvRyL3_K_E;87CLF3ARYjT*t-@0 literal 0 HcmV?d00001 -- 2.49.1 From ca993937003af3c890a325cda171eb7a8bfe0b31 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 21:17:54 -0400 Subject: [PATCH 24/80] Fix hyphenated needs references in CI gate expressions --- .gitea/workflows/cicd.yaml | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 2b470c5..28588f1 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -195,7 +195,7 @@ jobs: - name: Resolve head SHA id: meta run: | - echo "head_sha=${{ needs.publish-base.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT" - name: Minimal checkout for CICD build inputs env: @@ -233,7 +233,7 @@ jobs: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }} - BASE_HASH_FROM_BUILD: ${{ needs.publish-base.outputs.base_hash }} + BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }} run: | set -euo pipefail umask 077 @@ -521,9 +521,9 @@ jobs: - name: Confirm all source lanes passed run: | set -euo pipefail - backend_status="${{ needs.backend-tests.result }}" - precommit_status="${{ needs.precommit-tests.result }}" - frontend_status="${{ needs.frontend-tests.result }}" + backend_status="${{ needs['backend-tests'].result }}" + precommit_status="${{ needs['precommit-tests'].result }}" + frontend_status="${{ needs['frontend-tests'].result }}" xdoctest_status="${{ needs.xdoctest.result }}" echo "backend-tests=${backend_status}" @@ -1358,10 +1358,10 @@ jobs: - name: Confirm required production/tester images are complete run: | set -euo pipefail - backend_main_status="${{ needs.build-backend-main-image.result }}" - frontend_main_status="${{ needs.build-frontend-main-image.result }}" - integration_tester_status="${{ needs.build-integration-tester-image.result }}" - e2e_tester_status="${{ needs.build-e2e-tester-image.result }}" + backend_main_status="${{ needs['build-backend-main-image'].result }}" + frontend_main_status="${{ needs['build-frontend-main-image'].result }}" + integration_tester_status="${{ needs['build-integration-tester-image'].result }}" + e2e_tester_status="${{ needs['build-e2e-tester-image'].result }}" echo "build-backend-main-image=${backend_main_status}" echo "build-frontend-main-image=${frontend_main_status}" @@ -1618,9 +1618,9 @@ jobs: - name: Run runtime black-box integration checks via compose env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_digest_ref }} - INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} run: | set -euo pipefail set -o pipefail @@ -1834,13 +1834,13 @@ jobs: name: Integration Tests Failure Postmortem runs-on: ubuntu-act needs: integration-tests - if: always() && needs.integration-tests.result == 'failure' + if: always() && needs['integration-tests'].result == 'failure' timeout-minutes: 10 steps: - name: Integration postmortem diagnostics run: | echo "=== Integration Tests Postmortem ===" - echo "integration_tests_result=${{ needs.integration-tests.result }}" + echo "integration_tests_result=${{ needs['integration-tests'].result }}" uname -a || true cat /etc/os-release 2>/dev/null || true df -h || true @@ -1875,12 +1875,12 @@ jobs: env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build-backend-main-image.outputs.deployable_backend_digest_ref }} - DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.build-frontend-main-image.outputs.deployable_frontend_tag_ref }} - DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.build-frontend-main-image.outputs.deployable_frontend_digest_ref }} - INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} - E2E_TESTER_DIGEST_REF: ${{ needs.build-e2e-tester-image.outputs.e2e_tester_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} + DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }} + DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} + E2E_TESTER_DIGEST_REF: ${{ needs['build-e2e-tester-image'].outputs.e2e_tester_digest_ref }} run: | set -euo pipefail set -o pipefail @@ -2211,13 +2211,13 @@ jobs: name: E2E Tests Failure Postmortem runs-on: ubuntu-act needs: e2e-tests - if: always() && needs.e2e-tests.result == 'failure' + if: always() && needs['e2e-tests'].result == 'failure' timeout-minutes: 10 steps: - name: E2E postmortem diagnostics run: | echo "=== E2E Tests Postmortem ===" - echo "e2e_tests_result=${{ needs.e2e-tests.result }}" + echo "e2e_tests_result=${{ needs['e2e-tests'].result }}" uname -a || true cat /etc/os-release 2>/dev/null || true df -h || true -- 2.49.1 From 6f8913ba707f14e36f4e4d43d5d0a23e1f6df8e0 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 21:41:20 -0400 Subject: [PATCH 25/80] Harden CICD image refs against empty SHA/base hash --- .gitea/workflows/cicd.yaml | 41 ++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 28588f1..d5a16a3 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -239,6 +239,21 @@ jobs: umask 077 trap 'rm -f /tmp/ssh_key' EXIT + trim_spaces() { + printf '%s' "$1" | tr -d '[:space:]' + } + + is_hex() { + value="$1" + if [ -z "${value}" ]; then + return 1 + fi + case "${value}" in + (*[!0-9a-fA-F]*) return 1 ;; + (*) return 0 ;; + esac + } + echo "=== Pre-build disk telemetry ===" df -h || true if command -v docker >/dev/null 2>&1; then @@ -252,7 +267,25 @@ jobs: echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - BASE_HASH="${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}" + RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")" + if ! is_hex "${RESOLVED_HEAD_SHA}"; then + echo "❌ Invalid or empty HEAD_SHA resolved for build_cicd: '${HEAD_SHA:-}'" + exit 1 + fi + + BASE_HASH="$(trim_spaces "${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}")" + if ! is_hex "${BASE_HASH}"; then + BASE_HASH="$(./scripts/compute-cicd-base-hash.sh | tr -d '[:space:]')" + fi + + if ! is_hex "${BASE_HASH}"; then + echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}'" + exit 1 + fi + + echo "resolved_head_sha=${RESOLVED_HEAD_SHA}" + echo "resolved_base_hash=${BASE_HASH}" + BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" docker pull "${BASE_IMAGE}" @@ -262,14 +295,14 @@ jobs: docker build -f Dockerfile.cicd \ --secret id=ssh_private_key,src=/tmp/ssh_key \ --add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \ - --build-arg GITHUB_SHA="${HEAD_SHA}" \ + --build-arg GITHUB_SHA="${RESOLVED_HEAD_SHA}" \ --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ -t cicd:latest . docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" + docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" - *failure_diagnostics_step -- 2.49.1 From 93529b91ec1043eaf58ac929af5cc089c7adacb3 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 23:39:23 -0400 Subject: [PATCH 26/80] Use internal Playwright mirror for E2E tester builds --- .gitea/workflows/cicd.yaml | 18 ++++++++++-------- Dockerfile.e2e-tester | 3 ++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index d5a16a3..83eaaa2 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -864,9 +864,9 @@ jobs: run: | set -euo pipefail - PLAYWRIGHT_BASE_IMAGE="mcr.microsoft/playwright:v1.56.1-jammy" + PLAYWRIGHT_BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" - retry_pull_with_dns_probe() { + retry_base_pull() { image_ref="$1" attempts="${2:-6}" backoff="${3:-5}" @@ -875,13 +875,13 @@ jobs: while [ "${attempt}" -le "${attempts}" ]; do echo "base pull attempt ${attempt}/${attempts} for ${image_ref}" - if getent hosts mcr.microsoft >/dev/null 2>&1; then - echo "mcr.microsoft DNS resolution: ok" + if getent hosts "${GITEA_REGISTRY_HOST}" >/dev/null 2>&1; then + echo "${GITEA_REGISTRY_HOST} DNS resolution: ok" else - echo "mcr.microsoft DNS resolution: failed" + echo "${GITEA_REGISTRY_HOST} DNS resolution: failed" fi - timeout 10 curl -fsSIL "https://mcr.microsoft/v2/" >/dev/null || true + timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true if docker pull "${image_ref}"; then return 0 @@ -932,8 +932,10 @@ jobs: E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" - retry_pull_with_dns_probe "${PLAYWRIGHT_BASE_IMAGE}" 6 5 - DOCKER_BUILDKIT=0 docker build --pull=false -f Dockerfile.e2e-tester -t plex-playlist-e2e:"${HEAD_SHA}" . + retry_base_pull "${PLAYWRIGHT_BASE_IMAGE}" 6 5 + DOCKER_BUILDKIT=0 docker build --pull=false -f Dockerfile.e2e-tester \ + --build-arg PLAYWRIGHT_BASE_IMAGE="${PLAYWRIGHT_BASE_IMAGE}" \ + -t plex-playlist-e2e:"${HEAD_SHA}" . docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}" retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4 diff --git a/Dockerfile.e2e-tester b/Dockerfile.e2e-tester index 57fff55..a79f3ea 100644 --- a/Dockerfile.e2e-tester +++ b/Dockerfile.e2e-tester @@ -1,4 +1,5 @@ -FROM mcr.microsoft/playwright:v1.56.1-jammy +ARG PLAYWRIGHT_BASE_IMAGE=mcr.microsoft/playwright:v1.56.1-jammy +FROM ${PLAYWRIGHT_BASE_IMAGE} WORKDIR /workspace/frontend -- 2.49.1 From 062f19b4aa9058d2257d09a66cdebfbcf93bcc2f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 07:41:50 -0400 Subject: [PATCH 27/80] ci: retrigger workflow run -- 2.49.1 From e655540b48eb62b0cf7b48f9c8d9cdc47af87879 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 09:39:11 -0400 Subject: [PATCH 28/80] Remove residual runner-2 diagnostics drift handling --- .../collect_runner_diagnostics.xsh | 103 +++++++++++------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/scripts/gitea-actions/collect_runner_diagnostics.xsh b/scripts/gitea-actions/collect_runner_diagnostics.xsh index 3251a44..fc98e3d 100644 --- a/scripts/gitea-actions/collect_runner_diagnostics.xsh +++ b/scripts/gitea-actions/collect_runner_diagnostics.xsh @@ -4,7 +4,8 @@ # source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago" # source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234" -from datetime import datetime +from datetime import datetime, UTC +import subprocess hosts = [ "kankali.darkhelm.lan", @@ -16,53 +17,79 @@ hosts = [ since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago" trace = $ARGS[1] if len($ARGS) > 1 else "" -print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z") +print(f"runner-diagnostics-start={datetime.now(UTC).isoformat()}") print(f"since={since}") print(f"trace_filter={trace or 'none'}") -remote_script = f''' -set -eu -SINCE="{since}" -TRACE="{trace}" +remote_script = """ +import subprocess -echo "host=$(hostname -f 2>/dev/null || hostname)" -echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" -echo "kernel=$(uname -r 2>/dev/null || echo unknown)" +SINCE = __SINCE__ +TRACE = __TRACE__ -echo "=== system pressure ===" -free -h || true -df -h || true -uptime || true +def run(cmd: str): + proc = subprocess.run(cmd, shell=True, text=True, capture_output=True) + if proc.stdout: + print(proc.stdout, end="") + if proc.stderr: + print(proc.stderr, end="") + return proc.returncode -echo "=== docker runner containers ===" -docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match" +run('echo "host=$(hostname -f 2>/dev/null || hostname)"') +run('echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"') +run('echo "kernel=$(uname -r 2>/dev/null || echo unknown)"') -echo "=== runner container inspect ===" -for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do - echo "--- container=$c ---" - docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true - docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true -done +print('=== system pressure ===') +run('free -h || true') +run('df -h || true') +run('uptime || true') -echo "=== docker daemon recent log ===" -if command -v journalctl >/dev/null 2>&1; then - journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true -else - echo "journalctl_unavailable=true" -fi +print('=== docker runner containers ===') +ps_cmd = "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}' | grep -E 'gitea|runner|act'" +if run(ps_cmd) != 0: + print('no_runner_container_match') -echo "=== oom and kill signals ===" -dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true +print('=== runner container inspect ===') +names_cmd = "docker ps -a --format '{{.Names}}' | grep -E 'gitea|runner|act' || true" +names_proc = subprocess.run(names_cmd, shell=True, text=True, capture_output=True) +container_names = [line.strip() for line in (names_proc.stdout or '').splitlines() if line.strip()] -if [ -n "$TRACE" ]; then - echo "=== trace-filtered logs ===" - for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do - echo "--- trace search in container=$c ---" - docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true - done -fi -''' +for c in container_names: + print(f'--- container={c} ---') + run(f"docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' {c} || true") + run(f'docker logs --since "{SINCE}" --tail 200 {c} 2>&1 || true') + +print('=== docker daemon recent log ===') +if run('command -v journalctl >/dev/null 2>&1') == 0: + run(f'journalctl -u docker --since "{SINCE}" --no-pager -n 400 2>/dev/null || true') +else: + print('journalctl_unavailable=true') + +print('=== oom and kill signals ===') +run("dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true") + +if TRACE: + print('=== trace-filtered logs ===') + for c in container_names: + print(f'--- trace search in container={c} ---') + run(f'docker logs --since "{SINCE}" {c} 2>&1 | grep -F "{TRACE}" | tail -n 80 || true') +""" + +remote_script = remote_script.replace("__SINCE__", repr(since)).replace("__TRACE__", repr(trace)) for host in hosts: print(f"\n=== {host} ===") - ssh @(host) @(remote_script) + try: + result = subprocess.run( + ["ssh", host], + input=remote_script, + capture_output=True, + text=True, + check=False, + ) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="") + except Exception as exc: + print(f"runner_diagnostics_error host={host} error={exc}") -- 2.49.1 From d341005a268fc4d939715de2a8365ed73e05f329 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 11:14:18 -0400 Subject: [PATCH 29/80] ci: retrigger workflow run -- 2.49.1 From 26ea4ce136f1bfa5f2ca5d7367181270c65b056e Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 12:39:29 -0400 Subject: [PATCH 30/80] Retry base image registry operations on transient resets --- .gitea/workflows/cicd.yaml | 68 +++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 83eaaa2..3209289 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -623,14 +623,44 @@ jobs: run: | set -euo pipefail + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base" BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}" docker build -f Dockerfile.backend --target dependencies -t plex-playlist-backend-base:"${HEAD_SHA}" . docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}" - docker push "${BACKEND_BASE_TAG_REF}" - docker pull "${BACKEND_BASE_TAG_REF}" >/dev/null + retry_registry_op push "${BACKEND_BASE_TAG_REF}" 5 4 + retry_registry_op pull "${BACKEND_BASE_TAG_REF}" 5 3 BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)" if [ -z "${BACKEND_BASE_DIGEST_REF}" ]; then @@ -693,14 +723,44 @@ jobs: run: | set -euo pipefail + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base" FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}" docker build -f Dockerfile.frontend --target deps -t plex-playlist-frontend-base:"${HEAD_SHA}" . docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}" - docker push "${FRONTEND_BASE_TAG_REF}" - docker pull "${FRONTEND_BASE_TAG_REF}" >/dev/null + retry_registry_op push "${FRONTEND_BASE_TAG_REF}" 5 4 + retry_registry_op pull "${FRONTEND_BASE_TAG_REF}" 5 3 FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)" if [ -z "${FRONTEND_BASE_DIGEST_REF}" ]; then -- 2.49.1 From 2352152f502e03ff6cee4e9557df83767db1a36f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 14:00:44 -0400 Subject: [PATCH 31/80] Use ubuntu-act for non-build jobs and fix E2E backend probe --- .gitea/workflows/cicd.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 3209289..040d379 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -1445,7 +1445,7 @@ jobs: production-images-complete: name: Production Images Complete - runs-on: ubuntu-act-8gb + runs-on: ubuntu-act needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] if: always() timeout-minutes: 10 @@ -1476,7 +1476,7 @@ jobs: name: Runtime Images Failure Postmortem # Run on the broader runner pool so we can still capture diagnostics when # runtime_images fails during platform "Set up job" before any step executes. - runs-on: ubuntu-act-8gb + runs-on: ubuntu-act needs: [runtime_images, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] if: always() && (needs.runtime_images.result == 'failure' || needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') timeout-minutes: 10 @@ -1587,7 +1587,7 @@ jobs: name: Source Lanes Failure Postmortem # Capture follow-up diagnostics when any source lane dies during platform # setup before step logging is available. - runs-on: ubuntu-act-8gb + runs-on: ubuntu-act needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] if: always() && (needs['backend-tests'].result == 'failure' || needs['precommit-tests'].result == 'failure' || needs['frontend-tests'].result == 'failure' || needs.xdoctest.result == 'failure') timeout-minutes: 10 @@ -1692,8 +1692,8 @@ jobs: integration-tests: name: Runtime Black-Box Integration Tests - # Pin integration tests to high-memory worker to reduce setup-stage runner churn. - runs-on: ubuntu-act-8gb + # Use the broader runner pool; only docker build steps are 8gb-exclusive. + runs-on: ubuntu-act timeout-minutes: 20 needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image] steps: @@ -1949,7 +1949,7 @@ jobs: e2e-tests: name: End-to-End Tests # Use ubuntu-act runner pool for consistent availability. - runs-on: ubuntu-act-8gb + runs-on: ubuntu-act timeout-minutes: 45 needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] steps: @@ -2042,18 +2042,18 @@ jobs: 'import urllib.error' \ 'import urllib.request' \ 'endpoint_path = sys.argv[1]' \ - 'url = f\"http://127.0.0.1:8000{endpoint_path}\"' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ 'try:' \ ' with urllib.request.urlopen(url, timeout=2) as response:' \ - ' body = response.read().decode(\"utf-8\", errors=\"replace\")' \ + ' body = response.read().decode("utf-8", errors="replace")' \ ' print(response.status)' \ ' print(body)' \ 'except urllib.error.HTTPError as err:' \ - ' body = err.read().decode(\"utf-8\", errors=\"replace\")' \ + ' body = err.read().decode("utf-8", errors="replace")' \ ' print(err.code)' \ ' print(body)' \ 'except Exception:' \ - ' print(\"000\")' \ + ' print("000")' \ ' print("")')" "${endpoint_path}" } -- 2.49.1 From 6a4484a2f0c6be8078dd3d0563026cb64512d291 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 15:04:26 -0400 Subject: [PATCH 32/80] Retry backend main minimal checkout network operations --- .gitea/workflows/cicd.yaml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 040d379..4c74f34 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -1244,6 +1244,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -1251,12 +1275,13 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" + + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- \ .dockerignore \ -- 2.49.1 From e124045f1a1693d1e41fec370eff1bd391ae7754 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 16:22:26 -0400 Subject: [PATCH 33/80] Retry Corepack/Yarn bootstrap in frontend image build --- Dockerfile.frontend | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Dockerfile.frontend b/Dockerfile.frontend index 662b7ae..a58a56c 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -4,7 +4,24 @@ FROM node:24-alpine AS deps WORKDIR /app COPY frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml ./ -RUN corepack enable && yarn install --immutable +RUN set -eux; \ + retry() { \ + attempts="$1"; shift; \ + n=1; \ + while [ "$n" -le "$attempts" ]; do \ + if "$@"; then \ + return 0; \ + fi; \ + if [ "$n" -lt "$attempts" ]; then \ + sleep "$((3 * n))"; \ + fi; \ + n=$((n + 1)); \ + done; \ + return 1; \ + }; \ + corepack enable; \ + retry 5 corepack prepare yarn@4.10.3 --activate; \ + retry 5 yarn install --immutable FROM deps AS build -- 2.49.1 From 4b56d4e13bba629d0c7e8d961a8e883fb6592526 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 16:39:17 -0400 Subject: [PATCH 34/80] Fix Renovate endpoint to use Gitea API path --- .gitea/workflows/renovate.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 0202dd4..012559a 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -215,7 +215,7 @@ jobs: cat > renovate-config.js << 'EOF' module.exports = { platform: 'gitea', - endpoint: 'https://dogar.darkhelm.org', + endpoint: 'https://dogar.darkhelm.org/api/v1', gitAuthor: 'Renovate Bot ', repositories: ['DarkHelm.org/plex-playlist'], onboarding: false, @@ -248,12 +248,11 @@ jobs: RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" - RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" else API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" fi - export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + export RENOVATE_ENDPOINT="${API_ENDPOINT}" echo "Renovate endpoint: ${RENOVATE_ENDPOINT}" echo "Preflight API endpoint: ${API_ENDPOINT}" -- 2.49.1 From 5d37379aac7a68f8f4e0ce7fec118f4604d38b04 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 17:02:57 -0400 Subject: [PATCH 35/80] Standardize CI retries for checkout and runtime pulls --- .gitea/workflows/cicd.yaml | 419 +++++++++++++++++++++++++++++++++---- 1 file changed, 377 insertions(+), 42 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 4c74f34..5832fc4 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -75,6 +75,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -82,12 +106,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh else @@ -140,6 +164,36 @@ jobs: run: | set -euo pipefail + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -147,7 +201,7 @@ jobs: echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy" - docker pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" + retry_registry_op pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 5 3 PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}") @@ -159,8 +213,8 @@ jobs: docker tag cicd-base:latest "${BASE_REF_HASH}" docker tag cicd-base:latest "${BASE_REF_LATEST}" - docker push "${BASE_REF_HASH}" - docker push "${BASE_REF_LATEST}" + retry_registry_op push "${BASE_REF_HASH}" 5 4 + retry_registry_op push "${BASE_REF_LATEST}" 5 4 - &failure_diagnostics_step name: Failure diagnostics @@ -206,6 +260,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -213,12 +291,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh else @@ -254,6 +332,36 @@ jobs: esac } + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + echo "=== Pre-build disk telemetry ===" df -h || true if command -v docker >/dev/null 2>&1; then @@ -287,7 +395,7 @@ jobs: echo "resolved_base_hash=${BASE_HASH}" BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - docker pull "${BASE_IMAGE}" + retry_registry_op pull "${BASE_IMAGE}" 5 3 echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key chmod 600 /tmp/ssh_key @@ -299,10 +407,13 @@ jobs: --build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \ -t cicd:latest . - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" - docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" + CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest" + CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}" + + docker tag cicd:latest "${CICD_LATEST_REF}" + docker tag cicd:latest "${CICD_SHA_REF}" + retry_registry_op push "${CICD_LATEST_REF}" 5 4 + retry_registry_op push "${CICD_SHA_REF}" 5 4 - *failure_diagnostics_step @@ -595,6 +706,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -602,12 +737,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock else @@ -695,6 +830,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -702,12 +861,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml else @@ -795,6 +954,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -802,12 +985,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- Dockerfile.integration-tester else @@ -896,6 +1079,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -903,12 +1110,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- Dockerfile.e2e-tester else @@ -1051,6 +1258,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -1058,12 +1289,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- \ .dockerignore \ @@ -1380,6 +1611,30 @@ jobs: umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -1387,12 +1642,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git checkout FETCH_HEAD -- \ .dockerignore \ @@ -1838,6 +2093,34 @@ jobs: trap cleanup EXIT + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "pull" ]; then + if docker pull "${image_ref}"; then + return 0 + fi + else + return 1 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + { if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then echo "❌ Missing deployable backend image references from dispatch inputs" @@ -1846,8 +2129,8 @@ jobs: echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" @@ -2160,6 +2443,58 @@ jobs: trap cleanup EXIT + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "pull" ]; then + if docker pull "${image_ref}"; then + return 0 + fi + else + return 1 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then echo "❌ Missing deployable runtime image references from dispatch inputs" exit 1 @@ -2168,12 +2503,12 @@ jobs: mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" "${E2E_REPO_DIR}" - if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git -C "${E2E_REPO_DIR}" fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then git -C "${E2E_REPO_DIR}" checkout FETCH_HEAD -- frontend else @@ -2181,10 +2516,10 @@ jobs: fi echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" - docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" - docker pull "${DEPLOYABLE_FRONTEND_TAG_REF}" - docker pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" 5 3 DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" EXPECTED_BACKEND_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" @@ -2308,7 +2643,7 @@ jobs: -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \ -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ "${E2E_TESTER_DIGEST_REF}" \ - bash -lc "cd /workspace/frontend && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \ + bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; cd /workspace/frontend; retry_cmd 5 4 yarn install --immutable; yarn test:e2e --reporter=list --timeout=90000' \ 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} -- 2.49.1 From 548232afe28245d45fcb8b5f7bdad4df98a8a58b Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 17:19:19 -0400 Subject: [PATCH 36/80] Run Renovate via container instead of npm install --- .gitea/workflows/renovate.yml | 223 +++++++--------------------------- 1 file changed, 45 insertions(+), 178 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 012559a..20993d7 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -20,190 +20,44 @@ jobs: timeout-minutes: 90 steps: - - name: Setup Node.js for Renovate + - name: Prepare Renovate container image + env: + RENOVATE_IMAGE: ghcr.io/renovatebot/renovate:41 run: | - echo "=== Setting up Node.js 24 for Renovate ===" + set -euo pipefail - # Check existing Node.js - if command -v node &> /dev/null; then - echo "Current Node.js version: $(node --version)" - fi - if command -v npm &> /dev/null; then - echo "Current npm version: $(npm --version)" - fi + retry_cmd() { + attempts="${1:-3}" + backoff="${2:-10}" + shift 2 - # Aggressive cleanup of all Node.js/npm installations - echo "Performing complete Node.js cleanup..." - - # Stop any Node.js processes - sudo pkill -f node || true - - # Remove all package-managed Node.js installations - sudo apt-get remove -y --purge nodejs npm node || true - sudo apt-get autoremove -y --purge || true - - # Remove all manual installations and caches - sudo rm -rf /usr/local/bin/node* /usr/local/bin/npm* || true - sudo rm -rf /usr/local/lib/node* /usr/local/include/node* || true - sudo rm -rf ~/.npm ~/.nvm ~/.node* || true - sudo rm -rf /root/.npm /root/.nvm /root/.node* || true - sudo rm -rf /usr/share/nodejs || true - sudo rm -rf /etc/apt/sources.list.d/nodesource.list* || true - - # Clear npm environment variables that might conflict - unset npm_config_prefix npm_config_cache npm_config_globalconfig npm_config_init_module || true - - echo "✓ Cleanup completed" - - # Install Node.js 24 from NodeSource with error handling - echo "Installing Node.js 24..." - - # Remove any existing NodeSource repository - sudo rm -f /etc/apt/sources.list.d/nodesource.list || true - - # Add NodeSource repository - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - - - # Install with DEBIAN_FRONTEND to avoid interactive prompts - echo "Installing Node.js package..." - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs - - # Verify and fix installation - echo "=== Verifying Node.js Installation ===" - - # Check Node.js - if command -v node &> /dev/null; then - NODE_VERSION=$(node --version) - echo "✓ Node.js installed: $NODE_VERSION" - else - echo "❌ Node.js installation failed" - exit 1 - fi - - # Check npm and fix if needed - if command -v npm &> /dev/null && npm --version &> /dev/null; then - NPM_VERSION=$(npm --version) - echo "✓ npm working: $NPM_VERSION" - else - echo "⚠️ npm not working properly, reinstalling..." - - # Method 1: Try to fix npm with the bundled version - if [ -f "/usr/bin/node" ] && [ -f "/usr/lib/node_modules/npm/bin/npm-cli.js" ]; then - echo "Using bundled npm..." - sudo ln -sf /usr/lib/node_modules/npm/bin/npm-cli.js /usr/bin/npm || true - sudo chmod +x /usr/bin/npm || true - fi - - # Method 2: If that doesn't work, reinstall npm manually - if ! npm --version &> /dev/null; then - echo "Manual npm installation..." - curl -L https://www.npmjs.com/install.sh | sudo sh - fi - - # Method 3: Last resort - use npx to bootstrap npm - if ! npm --version &> /dev/null; then - echo "Using node to run npm directly..." - # Create npm wrapper script - echo '#!/bin/bash' | sudo tee /usr/bin/npm > /dev/null - echo 'exec /usr/bin/node /usr/lib/node_modules/npm/bin/npm-cli.js "$@"' | sudo tee -a /usr/bin/npm > /dev/null - sudo chmod +x /usr/bin/npm - fi - - # Final verification - if npm --version &> /dev/null; then - echo "✓ npm recovered successfully: $(npm --version)" - else - echo "❌ npm recovery failed" - exit 1 - fi - fi - - # Test npm basic functionality - echo "Testing npm functionality..." - if npm config get registry &> /dev/null; then - echo "✓ npm configuration accessible" - else - echo "⚠️ npm configuration issues, but continuing..." - fi - - # Check version compatibility for Renovate - NODE_VERSION=$(node --version | cut -d'v' -f2) - echo "=== Version Compatibility Check ===" - echo "Node.js version: $NODE_VERSION" - - if [[ $(echo "$NODE_VERSION 24.10.0" | awk '{print ($1 >= $2)}') == 1 ]]; then - echo "✅ Node.js version $NODE_VERSION meets Renovate latest requirements" - echo "RENOVATE_VERSION=latest" >> $GITHUB_ENV - else - echo "⚠️ Node.js version $NODE_VERSION - will use compatible Renovate version" - echo "RENOVATE_VERSION=40.3.2" >> $GITHUB_ENV - fi - - - name: Install Renovate - run: | - echo "=== Installing Renovate ===" - - # Set npm configuration for memory efficiency - npm config set fund false - npm config set audit false - npm config set progress false - npm config set maxsockets 3 - npm config set fetch-retry-mintimeout 2000 - npm config set fetch-retry-maxtimeout 10000 - - # Use the version determined in previous step - echo "Installing Renovate version: $RENOVATE_VERSION" - - # Check available memory - echo "Memory info:" - free -h || echo "free command not available" - - # Install with retry logic and memory-efficient options - for i in 1 2 3; do - echo "Renovate installation attempt $i/3..." - - # Clear npm cache and temp files - npm cache clean --force || true - rm -rf /tmp/npm-* || true - - # Use memory-efficient installation with longer timeout - if NODE_OPTIONS="--max-old-space-size=2048" timeout 600 npm install -g "renovate@$RENOVATE_VERSION" \ - --no-audit \ - --no-fund \ - --no-optional \ - --production \ - --maxsockets=3 \ - --fetch-timeout=30000; then - echo "✓ Renovate installation successful on attempt $i" - break - else - exit_code=$? - echo "⚠️ Renovate installation attempt $i failed (exit code: $exit_code)" - - # Check if it was killed due to memory - if [ $exit_code -eq 137 ] || [ $exit_code -eq 143 ]; then - echo " Process was killed (likely out of memory)" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 fi - if [ $i -eq 3 ]; then - echo "❌ All Renovate installation attempts failed" - echo "Debugging information:" - echo "Node.js version: $(node --version)" - echo "npm version: $(npm --version)" - echo "Memory status:" - free -h || echo "Memory info unavailable" - echo "npm config:" - npm config list || echo "npm config failed" - exit 1 + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" fi - echo "Waiting 30 seconds before retry..." - sleep 30 - fi - done + attempt=$((attempt + 1)) + done - # Verify Renovate installation - echo "✓ Renovate version: $(renovate --version)" - echo "✓ Renovate location: $(which renovate)" + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + echo "=== Preparing Renovate container ===" + echo "Using Renovate image: ${RENOVATE_IMAGE}" + free -h || true + + retry_cmd 3 15 docker pull "${RENOVATE_IMAGE}" + docker run --rm "${RENOVATE_IMAGE}" --version + + echo "RENOVATE_IMAGE=${RENOVATE_IMAGE}" >> "$GITHUB_ENV" - name: Configure Renovate for Gitea run: | @@ -330,7 +184,20 @@ jobs: unset RENOVATE_DRY_RUN fi - renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + -e LOG_LEVEL \ + -e NODE_TLS_REJECT_UNAUTHORIZED \ + -e RENOVATE_TOKEN \ + -e RENOVATE_PLATFORM \ + -e RENOVATE_ENDPOINT \ + -e RENOVATE_DRY_RUN \ + -e RENOVATE_CONFIG_FILE=/work/renovate-config.js \ + "${RENOVATE_IMAGE}" \ + --platform "${RENOVATE_PLATFORM}" \ + --endpoint "${RENOVATE_ENDPOINT}" \ + DarkHelm.org/plex-playlist | tee /tmp/renovate.log echo "✓ Renovate execution completed" -- 2.49.1 From 3fed66c2a0d469defb5a94078ceae8672d491996 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 19:59:09 -0400 Subject: [PATCH 37/80] Fix Renovate config mount path in container run --- .gitea/workflows/renovate.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 20993d7..e9cfb7f 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -63,10 +63,14 @@ jobs: run: | echo "=== Configuring Renovate for Gitea ===" + CONFIG_DIR="${RUNNER_TEMP:-/tmp}/renovate-config" + CONFIG_FILE="${CONFIG_DIR}/renovate-config.js" + mkdir -p "${CONFIG_DIR}" + # Renovate reads RENOVATE_TOKEN, RENOVATE_PLATFORM, RENOVATE_ENDPOINT, # and RENOVATE_GIT_AUTHOR directly from the environment at runtime. # Keeping the config file minimal avoids repeated migration warnings. - cat > renovate-config.js << 'EOF' + cat > "${CONFIG_FILE}" << 'EOF' module.exports = { platform: 'gitea', endpoint: 'https://dogar.darkhelm.org/api/v1', @@ -80,6 +84,7 @@ jobs: }; EOF + echo "RENOVATE_CONFIG_FILE_HOST=${CONFIG_FILE}" >> "$GITHUB_ENV" echo "✓ Renovate configuration created" - name: Run Renovate @@ -89,7 +94,7 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} - RENOVATE_CONFIG_FILE: renovate-config.js + RENOVATE_CONFIG_FILE: /renovate-config/renovate-config.js RENOVATE_PLATFORM: gitea RENOVATE_ENDPOINT: https://dogar.darkhelm.org RENOVATE_ALLOW_INSECURE_TLS: "true" @@ -184,16 +189,20 @@ jobs: unset RENOVATE_DRY_RUN fi + if [ ! -f "${RENOVATE_CONFIG_FILE_HOST:-}" ]; then + echo "❌ Renovate config file missing on host: ${RENOVATE_CONFIG_FILE_HOST:-unset}" + exit 1 + fi + docker run --rm \ - -v "$PWD:/work" \ - -w /work \ + -v "$(dirname "${RENOVATE_CONFIG_FILE_HOST}"):/renovate-config:ro" \ -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e RENOVATE_TOKEN \ -e RENOVATE_PLATFORM \ -e RENOVATE_ENDPOINT \ -e RENOVATE_DRY_RUN \ - -e RENOVATE_CONFIG_FILE=/work/renovate-config.js \ + -e RENOVATE_CONFIG_FILE \ "${RENOVATE_IMAGE}" \ --platform "${RENOVATE_PLATFORM}" \ --endpoint "${RENOVATE_ENDPOINT}" \ -- 2.49.1 From 231074ae405b32c372a449686471d65b87d614b1 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Wed, 8 Jul 2026 20:03:50 -0400 Subject: [PATCH 38/80] Fix E2E frontend readiness probe to use compose service check --- .gitea/workflows/cicd.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 5832fc4..b065729 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2296,7 +2296,6 @@ jobs: DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" E2E_REPO_DIR="$(mktemp -d)" - FRONTEND_URL_RUNNER="http://frontend:80" if ! command -v docker >/dev/null 2>&1; then echo "❌ Docker is required" @@ -2619,11 +2618,13 @@ jobs: for i in $(seq 1 30); do frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)" if [ "${frontend_running_count}" -gt 0 ]; then - frontend_root_code="$(fetch_url_from_backend "${FRONTEND_URL_RUNNER}" /tmp/e2e-frontend-root.html)" - if [ "${frontend_root_code}" = "200" ]; then - frontend_ready=true - break - fi + frontend_root_code="$(fetch_http "frontend" "/" /tmp/e2e-frontend-root.html 80)" + case "${frontend_root_code}" in + 2*|3*) + frontend_ready=true + break + ;; + esac fi sleep 2 done -- 2.49.1 From 758967e4e608e555f6fdc14f660e4568d0b397f8 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 00:24:18 -0400 Subject: [PATCH 39/80] Fix E2E frontend probe and remove Renovate config mount dependency --- .gitea/workflows/cicd.yaml | 3 ++- .gitea/workflows/renovate.yml | 48 ++++++++++++----------------------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index b065729..995e587 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2296,6 +2296,7 @@ jobs: DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" E2E_REPO_DIR="$(mktemp -d)" + FRONTEND_URL_COMPOSE="http://frontend:80" if ! command -v docker >/dev/null 2>&1; then echo "❌ Docker is required" @@ -2618,7 +2619,7 @@ jobs: for i in $(seq 1 30); do frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)" if [ "${frontend_running_count}" -gt 0 ]; then - frontend_root_code="$(fetch_http "frontend" "/" /tmp/e2e-frontend-root.html 80)" + frontend_root_code="$(fetch_url_from_backend "${FRONTEND_URL_COMPOSE}" /tmp/e2e-frontend-root.html)" case "${frontend_root_code}" in 2*|3*) frontend_ready=true diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index e9cfb7f..66128eb 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -62,30 +62,7 @@ jobs: - name: Configure Renovate for Gitea run: | echo "=== Configuring Renovate for Gitea ===" - - CONFIG_DIR="${RUNNER_TEMP:-/tmp}/renovate-config" - CONFIG_FILE="${CONFIG_DIR}/renovate-config.js" - mkdir -p "${CONFIG_DIR}" - - # Renovate reads RENOVATE_TOKEN, RENOVATE_PLATFORM, RENOVATE_ENDPOINT, - # and RENOVATE_GIT_AUTHOR directly from the environment at runtime. - # Keeping the config file minimal avoids repeated migration warnings. - cat > "${CONFIG_FILE}" << 'EOF' - module.exports = { - platform: 'gitea', - endpoint: 'https://dogar.darkhelm.org/api/v1', - gitAuthor: 'Renovate Bot ', - repositories: ['DarkHelm.org/plex-playlist'], - onboarding: false, - requireConfig: 'required', - extends: ['local>DarkHelm.org/plex-playlist'], - prConcurrentLimit: 3, - branchConcurrentLimit: 5, - }; - EOF - - echo "RENOVATE_CONFIG_FILE_HOST=${CONFIG_FILE}" >> "$GITHUB_ENV" - echo "✓ Renovate configuration created" + echo "✓ Renovate runtime configuration will be passed through environment and CLI flags" - name: Run Renovate env: @@ -94,10 +71,14 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} - RENOVATE_CONFIG_FILE: /renovate-config/renovate-config.js RENOVATE_PLATFORM: gitea RENOVATE_ENDPOINT: https://dogar.darkhelm.org RENOVATE_ALLOW_INSECURE_TLS: "true" + RENOVATE_GIT_AUTHOR: Renovate Bot + RENOVATE_ONBOARDING: "false" + RENOVATE_REQUIRE_CONFIG: required + RENOVATE_PR_CONCURRENT_LIMIT: "3" + RENOVATE_BRANCH_CONCURRENT_LIMIT: "5" LOG_LEVEL: info run: | echo "=== Running Renovate Bot ===" @@ -189,23 +170,26 @@ jobs: unset RENOVATE_DRY_RUN fi - if [ ! -f "${RENOVATE_CONFIG_FILE_HOST:-}" ]; then - echo "❌ Renovate config file missing on host: ${RENOVATE_CONFIG_FILE_HOST:-unset}" - exit 1 - fi - docker run --rm \ - -v "$(dirname "${RENOVATE_CONFIG_FILE_HOST}"):/renovate-config:ro" \ -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e RENOVATE_TOKEN \ -e RENOVATE_PLATFORM \ -e RENOVATE_ENDPOINT \ -e RENOVATE_DRY_RUN \ - -e RENOVATE_CONFIG_FILE \ + -e RENOVATE_GIT_AUTHOR \ + -e RENOVATE_ONBOARDING \ + -e RENOVATE_REQUIRE_CONFIG \ + -e RENOVATE_PR_CONCURRENT_LIMIT \ + -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ "${RENOVATE_IMAGE}" \ --platform "${RENOVATE_PLATFORM}" \ --endpoint "${RENOVATE_ENDPOINT}" \ + --git-author "${RENOVATE_GIT_AUTHOR}" \ + --onboarding="${RENOVATE_ONBOARDING}" \ + --require-config "${RENOVATE_REQUIRE_CONFIG}" \ + --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" \ + --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" \ DarkHelm.org/plex-playlist | tee /tmp/renovate.log echo "✓ Renovate execution completed" -- 2.49.1 From acb446c30f7445af3065d5856ef08ad7a799ee4c Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 01:42:27 -0400 Subject: [PATCH 40/80] Retry frontend-base registry login and harden Renovate endpoint auth --- .gitea/workflows/cicd.yaml | 26 ++++++++++++- .gitea/workflows/renovate.yml | 73 ++++++++++++++++++++++++----------- 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 995e587..6dcba80 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -882,6 +882,30 @@ jobs: run: | set -euo pipefail + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + retry_registry_op() { op_name="$1" image_ref="$2" @@ -912,7 +936,7 @@ jobs: return 1 } - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + retry_cmd 5 3 sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin' FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base" FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}" diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 66128eb..4f96b9e 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -88,14 +88,27 @@ jobs: RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + BASE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" else API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" + BASE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" fi - export RENOVATE_ENDPOINT="${API_ENDPOINT}" - echo "Renovate endpoint: ${RENOVATE_ENDPOINT}" + export RENOVATE_ENDPOINT="${BASE_ENDPOINT}" + echo "Renovate endpoint (primary): ${RENOVATE_ENDPOINT}" + echo "Renovate endpoint (fallback): ${API_ENDPOINT}" echo "Preflight API endpoint: ${API_ENDPOINT}" + ENDPOINT_HOST="$(printf '%s' "${RENOVATE_ENDPOINT_EFFECTIVE}" | sed -E 's#^https?://([^/]+).*$#\1#')" + ENDPOINT_IP="$(getent hosts "${ENDPOINT_HOST}" | awk '{print $1; exit}' || true)" + DOCKER_ADD_HOST_ARG="" + if [ -n "${ENDPOINT_IP}" ]; then + DOCKER_ADD_HOST_ARG="--add-host ${ENDPOINT_HOST}:${ENDPOINT_IP}" + echo "Renovate container host mapping: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" + else + echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host" + fi + CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then echo "⚠ Renovate insecure TLS mode enabled for self-signed certificate endpoint" @@ -170,27 +183,41 @@ jobs: unset RENOVATE_DRY_RUN fi - docker run --rm \ - -e LOG_LEVEL \ - -e NODE_TLS_REJECT_UNAUTHORIZED \ - -e RENOVATE_TOKEN \ - -e RENOVATE_PLATFORM \ - -e RENOVATE_ENDPOINT \ - -e RENOVATE_DRY_RUN \ - -e RENOVATE_GIT_AUTHOR \ - -e RENOVATE_ONBOARDING \ - -e RENOVATE_REQUIRE_CONFIG \ - -e RENOVATE_PR_CONCURRENT_LIMIT \ - -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ - "${RENOVATE_IMAGE}" \ - --platform "${RENOVATE_PLATFORM}" \ - --endpoint "${RENOVATE_ENDPOINT}" \ - --git-author "${RENOVATE_GIT_AUTHOR}" \ - --onboarding="${RENOVATE_ONBOARDING}" \ - --require-config "${RENOVATE_REQUIRE_CONFIG}" \ - --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" \ - --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" \ - DarkHelm.org/plex-playlist | tee /tmp/renovate.log + run_renovate_with_endpoint() { + endpoint="$1" + export RENOVATE_ENDPOINT="${endpoint}" + echo "Running Renovate with endpoint: ${RENOVATE_ENDPOINT}" + + # shellcheck disable=SC2086 + docker run --rm ${DOCKER_ADD_HOST_ARG} \ + -e LOG_LEVEL \ + -e NODE_TLS_REJECT_UNAUTHORIZED \ + -e RENOVATE_TOKEN \ + -e RENOVATE_PLATFORM \ + -e RENOVATE_ENDPOINT \ + -e RENOVATE_DRY_RUN \ + -e RENOVATE_GIT_AUTHOR \ + -e RENOVATE_ONBOARDING \ + -e RENOVATE_REQUIRE_CONFIG \ + -e RENOVATE_PR_CONCURRENT_LIMIT \ + -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ + "${RENOVATE_IMAGE}" \ + --platform "${RENOVATE_PLATFORM}" \ + --endpoint "${RENOVATE_ENDPOINT}" \ + --git-author "${RENOVATE_GIT_AUTHOR}" \ + --onboarding="${RENOVATE_ONBOARDING}" \ + --require-config "${RENOVATE_REQUIRE_CONFIG}" \ + --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" \ + --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" \ + DarkHelm.org/plex-playlist + } + + if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then + : + else + echo "⚠ Renovate run failed with primary endpoint. Retrying with API endpoint form..." + run_renovate_with_endpoint "${API_ENDPOINT}" 2>&1 | tee /tmp/renovate.log + fi echo "✓ Renovate execution completed" -- 2.49.1 From aa1c1b79005084df6bf3e74477872a66ed12828b Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 09:11:47 -0400 Subject: [PATCH 41/80] Use nginx-side readiness probe for E2E frontend --- .gitea/workflows/cicd.yaml | 50 +++----------------------------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 6dcba80..c1f4a39 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2320,7 +2320,6 @@ jobs: DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist" E2E_REPO_DIR="$(mktemp -d)" - FRONTEND_URL_COMPOSE="http://frontend:80" if ! command -v docker >/dev/null 2>&1; then echo "❌ Docker is required" @@ -2412,44 +2411,6 @@ jobs: echo "${http_code}" } - fetch_url_from_backend() { - target_url="$1" - output_file="$2" - probe_tmp="$(mktemp)" - - if ! docker run --rm \ - --network "${COMPOSE_PROJECT_NAME}_default" \ - "${INTEGRATION_TESTER_DIGEST_REF}" \ - python -c "$(printf '%s\n' \ - 'import sys' \ - 'import urllib.error' \ - 'import urllib.request' \ - 'target_url = sys.argv[1]' \ - 'try:' \ - ' with urllib.request.urlopen(target_url, timeout=2) as response:' \ - ' body = response.read().decode(\"utf-8\", errors=\"replace\")' \ - ' print(response.status)' \ - ' print(body)' \ - 'except urllib.error.HTTPError as err:' \ - ' body = err.read().decode(\"utf-8\", errors=\"replace\")' \ - ' print(err.code)' \ - ' print(body)' \ - 'except Exception:' \ - ' print(\"000\")' \ - ' print("")')" \ - "${target_url}" >"${probe_tmp}" 2>/dev/null; then - : >"${output_file}" - rm -f "${probe_tmp}" - echo "000" - return 0 - fi - - http_code="$(head -n 1 "${probe_tmp}")" - tail -n +2 "${probe_tmp}" >"${output_file}" - rm -f "${probe_tmp}" - echo "${http_code}" - } - dump_failure_context() { echo "=== Runtime E2E Failure Context ===" docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true @@ -2643,13 +2604,10 @@ jobs: for i in $(seq 1 30); do frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)" if [ "${frontend_running_count}" -gt 0 ]; then - frontend_root_code="$(fetch_url_from_backend "${FRONTEND_URL_COMPOSE}" /tmp/e2e-frontend-root.html)" - case "${frontend_root_code}" in - 2*|3*) - frontend_ready=true - break - ;; - esac + if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T frontend sh -lc 'wget -q -T 2 -O /dev/null http://127.0.0.1/'; then + frontend_ready=true + break + fi fi sleep 2 done -- 2.49.1 From d3890d572690b1f274c1c7218d1ddc2b269390a3 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 09:48:36 -0400 Subject: [PATCH 42/80] Force Renovate token-only auth for Gitea --- .gitea/workflows/renovate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 4f96b9e..5e9c09c 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -171,6 +171,7 @@ jobs: SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" + export RENOVATE_HOST_RULES="[{\"matchHost\":\"${BASE_ENDPOINT}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" @@ -193,6 +194,7 @@ jobs: -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e RENOVATE_TOKEN \ + -e RENOVATE_HOST_RULES \ -e RENOVATE_PLATFORM \ -e RENOVATE_ENDPOINT \ -e RENOVATE_DRY_RUN \ -- 2.49.1 From f24de1dcab0997d629e4cb0b4f9b5648b862762f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 10:55:19 -0400 Subject: [PATCH 43/80] Instrument Renovate container auth preflight --- .gitea/workflows/renovate.yml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 5e9c09c..82dcafc 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -79,7 +79,7 @@ jobs: RENOVATE_REQUIRE_CONFIG: required RENOVATE_PR_CONCURRENT_LIMIT: "3" RENOVATE_BRANCH_CONCURRENT_LIMIT: "5" - LOG_LEVEL: info + LOG_LEVEL: debug run: | echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" @@ -116,6 +116,24 @@ jobs: export NODE_TLS_REJECT_UNAUTHORIZED=0 fi + VERSION_STATUS=$(curl -sS -o /tmp/renovate-version-check.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + "${API_ENDPOINT}/version" || true) + PLATFORM_VERSION="" + if [ "${VERSION_STATUS}" = "200" ]; then + PLATFORM_VERSION=$(sed -n 's/.*"version":"\([^"]*\)".*/\1/p' /tmp/renovate-version-check.json | head -n 1) + if [ -n "${PLATFORM_VERSION}" ]; then + export RENOVATE_X_PLATFORM_VERSION="${PLATFORM_VERSION}" + echo "✓ Gitea version preflight passed: ${RENOVATE_X_PLATFORM_VERSION}" + else + echo "⚠ Gitea version preflight returned 200 but version could not be parsed" + cat /tmp/renovate-version-check.json || true + fi + else + echo "⚠ Gitea version preflight failed with status ${VERSION_STATUS}" + cat /tmp/renovate-version-check.json || true + fi + select_token_with_repo_access() { for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do candidate_value="${!candidate_name:-}" @@ -176,6 +194,17 @@ jobs: echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" + echo "=== Renovate container connectivity preflight ===" + # Prove whether the same image/container path Renovate uses can reach Gitea. + # If this fails, the problem is container network/TLS resolution, not Renovate config. + # shellcheck disable=SC2086 + docker run --rm ${DOCKER_ADD_HOST_ARG} \ + -e API_ENDPOINT="${API_ENDPOINT}" \ + -e RENOVATE_TOKEN \ + -e NODE_TLS_REJECT_UNAUTHORIZED \ + "${RENOVATE_IMAGE}" \ + node -e 'const endpoints = [["version", `${process.env.API_ENDPOINT}/version`, {}], ["user", `${process.env.API_ENDPOINT}/user`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` } }]]; let failed = false; (async () => { for (const [name, url, options] of endpoints) { try { const response = await fetch(url, options); console.log(`container_${name}_status=${response.status}`); if (!response.ok) { failed = true; console.log(`container_${name}_body_start`); console.log((await response.text()).slice(0, 1000)); console.log(`container_${name}_body_end`); } } catch (error) { failed = true; console.log(`container_${name}_error=${error instanceof Error ? error.message : String(error)}`); } } if (failed) { process.exit(1); } })();' + # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then export RENOVATE_DRY_RUN="full" @@ -195,6 +224,7 @@ jobs: -e NODE_TLS_REJECT_UNAUTHORIZED \ -e RENOVATE_TOKEN \ -e RENOVATE_HOST_RULES \ + -e RENOVATE_X_PLATFORM_VERSION \ -e RENOVATE_PLATFORM \ -e RENOVATE_ENDPOINT \ -e RENOVATE_DRY_RUN \ -- 2.49.1 From 9731fee9bea3d2a5d8d1072702d2a457c6326188 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 13:16:16 -0400 Subject: [PATCH 44/80] Harden runner diagnostics and warm image caches --- README.md | 7 +- docs/GITEA_ACTIONS_TROUBLESHOOTING.md | 8 +- scripts/gitea-actions/check_runner_images.xsh | 61 +++++++++++--- .../gitea-actions/deploy-runner-config.xsh | 30 +++++++ .../gitea-actions/diagnose_runner_startup.xsh | 51 ++++++++---- scripts/gitea-actions/rollout-one-runner.xsh | 18 +++++ .../runner-prechange-capture.xsh | 80 ++++++++++++------- 7 files changed, 196 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index eac02b2..d9eb8b1 100644 --- a/README.md +++ b/README.md @@ -157,9 +157,14 @@ plex-playlist/ For Gitea Actions runner image mirror maintenance, use: - `source scripts/gitea-actions/repair_runner_mirror.xsh` -- `source scripts/gitea-actions/check_runner_images.xsh` +- `source scripts/gitea-actions/check_runner_images.xsh` (checks ubuntu:22.04, GHCR act image, local mirror, and Renovate image) - `source scripts/gitea-actions/collect_runner_diagnostics.xsh [since] [trace_id]` +For setup-stage failures with sparse logs, use: + +- `source scripts/gitea-actions/runner-prechange-capture.xsh [tail_lines]` +- `source scripts/gitea-actions/diagnose_runner_startup.xsh` + For full troubleshooting context, see `docs/GITEA_ACTIONS_TROUBLESHOOTING.md`. ## Environment Variables diff --git a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md index 54803c3..15bc1c0 100644 --- a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md +++ b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md @@ -178,7 +178,13 @@ Automation scripts for this workflow live in `scripts/gitea-actions/`: Repairs the mirror by pulling from GHCR, tagging/pushing to local registry, and verifying tag presence on each runner host. - `scripts/gitea-actions/check_runner_images.xsh` - Verifies host-by-host image presence for both upstream (`GHCR`) and mirrored (`MIRROR`) tags. + Verifies host-by-host image presence and pullability for `ubuntu:22.04`, upstream act image (`GHCR`), mirrored act image (`MIRROR`), and Renovate image. + +- `scripts/gitea-actions/runner-prechange-capture.xsh` + Captures restart counts, OOM flags, memory state, and runner logs across all hosts. + +- `scripts/gitea-actions/diagnose_runner_startup.xsh` + Focused startup diagnostics for runner containers and mirror pull behavior. Run from repository root (xonsh): diff --git a/scripts/gitea-actions/check_runner_images.xsh b/scripts/gitea-actions/check_runner_images.xsh index 5432232..c1d43e6 100644 --- a/scripts/gitea-actions/check_runner_images.xsh +++ b/scripts/gitea-actions/check_runner_images.xsh @@ -5,26 +5,63 @@ hosts = [ "pi-desktop.darkhelm.lan", ] +mirror_image = "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest" +ghcr_runner_image = "ghcr.io/catthehacker/ubuntu:act-latest" +renovate_image = "ghcr.io/renovatebot/renovate:41" +python_image = "python:3.14-slim" +node_image = "node:20-bookworm-slim" + # Check both upstream and mirrored tags on each runner host. remote_script = """ -r = !(docker image inspect ubuntu:22.04 > /dev/null 2> /dev/null) -print("UBUNTU22:present" if r.returncode == 0 else "UBUNTU22:missing") -r = !(docker image inspect ghcr.io/catthehacker/ubuntu:act-latest > /dev/null 2> /dev/null) -print("GHCR:present" if r.returncode == 0 else "GHCR:missing") -r = !(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null) -print("MIRROR_LOCAL:present" if r.returncode == 0 else "MIRROR_LOCAL:missing") +def local_state(label, image): + r = !(docker image inspect @(image) > /dev/null 2> /dev/null) + print(f"{label}:present" if r.returncode == 0 else f"{label}:missing") + +local_state("UBUNTU22", "ubuntu:22.04") +local_state("PYTHON", "__PYTHON_IMAGE__") +local_state("NODE", "__NODE_IMAGE__") +local_state("GHCR", "__GHCR_RUNNER_IMAGE__") +local_state("MIRROR_LOCAL", "__MIRROR_IMAGE__") +local_state("RENOVATE", "__RENOVATE_IMAGE__") r = !(docker info 2> /dev/null | grep -qi "kankali.darkhelm.lan:3001") print("REGISTRY_CONFIG:ok" if r.returncode == 0 else "REGISTRY_CONFIG:missing") -pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-pull.log 2>&1) -if pull.returncode == 0: - print("MIRROR_REMOTE:pull-ok") -else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-pull.log) - print("MIRROR_REMOTE:https-mismatch" if mismatch.returncode == 0 else "MIRROR_REMOTE:pull-failed") +def remote_pull(label, image, log_path): + pull = !(docker pull @(image) > @(log_path) 2>&1) + if pull.returncode == 0: + print(f"{label}:pull-ok") + return + mismatch = !(grep -qi "http response to https client" @(log_path)) + if mismatch.returncode == 0: + print(f"{label}:https-mismatch") + else: + print(f"{label}:pull-failed") + +remote_pull("UBUNTU22_REMOTE", "ubuntu:22.04", "/tmp/ubuntu22-pull.log") +remote_pull("PYTHON_REMOTE", "__PYTHON_IMAGE__", "/tmp/python-pull.log") +remote_pull("NODE_REMOTE", "__NODE_IMAGE__", "/tmp/node-pull.log") +remote_pull("GHCR_REMOTE", "__GHCR_RUNNER_IMAGE__", "/tmp/ghcr-runner-pull.log") +remote_pull("MIRROR_REMOTE", "__MIRROR_IMAGE__", "/tmp/mirror-pull.log") +remote_pull("RENOVATE_REMOTE", "__RENOVATE_IMAGE__", "/tmp/renovate-pull.log") + +local_state("UBUNTU22_AFTER_PULL", "ubuntu:22.04") +local_state("PYTHON_AFTER_PULL", "__PYTHON_IMAGE__") +local_state("NODE_AFTER_PULL", "__NODE_IMAGE__") +local_state("GHCR_AFTER_PULL", "__GHCR_RUNNER_IMAGE__") +local_state("MIRROR_AFTER_PULL", "__MIRROR_IMAGE__") +local_state("RENOVATE_AFTER_PULL", "__RENOVATE_IMAGE__") """ +remote_script = ( + remote_script + .replace("__MIRROR_IMAGE__", mirror_image) + .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image) + .replace("__RENOVATE_IMAGE__", renovate_image) + .replace("__PYTHON_IMAGE__", python_image) + .replace("__NODE_IMAGE__", node_image) +) + for host in hosts: print(f"\n=== {host} ===") ssh @(host) @(remote_script) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 8186902..8cc067c 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -18,6 +18,15 @@ HOSTS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +WARMUP_IMAGES = [ + "ubuntu:22.04", + "python:3.14-slim", + "node:20-bookworm-slim", + "ghcr.io/catthehacker/ubuntu:act-latest", + "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", + "ghcr.io/renovatebot/renovate:41", +] + DEFAULT_MEM_FRACTION = 0.75 DEFAULT_JOB_MEM_FRACTION = 0.9 DEFAULT_HOST_MEM_SHARE = 0.25 @@ -258,6 +267,20 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr +def warm_runner_images(host): + pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) + rc, out = ssh( + host, + ( + "set -e\n" + "echo 'warming_runner_images_start'\n" + f"{pulls}\n" + "echo 'warming_runner_images_done'\n" + ), + ) + return rc, out + + def service_block_bounds(lines, start_index): service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" ")) block_start = start_index + 1 @@ -743,6 +766,13 @@ with tempfile.TemporaryDirectory() as tmpdir: continue print(out.strip()) + rc, out = warm_runner_images(host) + if rc != 0: + print(out.strip()) + print("Failed to warm runner images on host") + continue + print(out.strip()) + rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'") print("mounts:", out.strip()) diff --git a/scripts/gitea-actions/diagnose_runner_startup.xsh b/scripts/gitea-actions/diagnose_runner_startup.xsh index 8339c98..70ec106 100755 --- a/scripts/gitea-actions/diagnose_runner_startup.xsh +++ b/scripts/gitea-actions/diagnose_runner_startup.xsh @@ -10,6 +10,8 @@ hosts = { } mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest' +ghcr_runner_image = 'ghcr.io/catthehacker/ubuntu:act-latest' +renovate_image = 'ghcr.io/renovatebot/renovate:41' do_fix = '--fix' in sys.argv[1:] remote_diag = f""" @@ -19,6 +21,9 @@ print('[runner_containers]') r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') print(str(r.out).strip()) +r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}}') +runner_names = [line.strip() for line in str(r.out).splitlines() if line.strip()] + print('[registry_config]') r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001') print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing') @@ -36,31 +41,47 @@ else: tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log) print(str(tail_out.out).strip()) -print('[recent_logs_runner1]') -log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1) -log1_lines = [ - line for line in str(log1.out).splitlines() - if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) -] -print('\\n'.join(log1_lines[-60:])) +if not runner_names: + print('[recent_logs] no-runner-containers-found') -print('[recent_logs_runner2]') -log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1) -log2_lines = [ - line for line in str(log2.out).splitlines() - if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) -] -print('\\n'.join(log2_lines[-60:])) +for name in runner_names: + print(f'[recent_logs_{name}]') + log = !(docker logs --tail 60 @(name) 2>&1) + log_lines = [ + line for line in str(log.out).splitlines() + if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline', 'no route to host', 'network is unreachable']) + ] + print('\\n'.join(log_lines[-60:])) """ remote_fix = """ print('[fix] restarting runners') -$(docker compose up -d --force-recreate runner1 runner2) +r = !(docker compose up -d --force-recreate act_runner_1 act_runner_2 > /tmp/runner-restart.log 2>&1) +if r.returncode != 0: + r = !(docker compose up -d --force-recreate runner1 runner2 > /tmp/runner-restart.log 2>&1) + if r.returncode != 0: + tail_out = !(tail -n 60 /tmp/runner-restart.log) + print(str(tail_out.out).strip()) + raise SystemExit(1) $(sleep 3) +print('[fix] warming images') +$(docker pull ubuntu:22.04) +$(docker pull python:3.14-slim) +$(docker pull node:20-bookworm-slim) +$(docker pull __GHCR_RUNNER_IMAGE__) +$(docker pull __MIRROR_IMAGE__) +$(docker pull __RENOVATE_IMAGE__) r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') print(str(r.out).strip()) """ +remote_fix = ( + remote_fix + .replace("__MIRROR_IMAGE__", mirror_image) + .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image) + .replace("__RENOVATE_IMAGE__", renovate_image) +) + for host, path in hosts.items(): print(f"\n===== {host} =====") cmd = f"cd {path}\n{remote_diag}" diff --git a/scripts/gitea-actions/rollout-one-runner.xsh b/scripts/gitea-actions/rollout-one-runner.xsh index afb5578..c97a8ef 100755 --- a/scripts/gitea-actions/rollout-one-runner.xsh +++ b/scripts/gitea-actions/rollout-one-runner.xsh @@ -13,6 +13,15 @@ HOST_PATHS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +WARMUP_IMAGES = [ + "ubuntu:22.04", + "python:3.14-slim", + "node:20-bookworm-slim", + "ghcr.io/catthehacker/ubuntu:act-latest", + "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", + "ghcr.io/renovatebot/renovate:41", +] + def usage(exit_code=1): print("Usage: xonsh rollout-one-runner.xsh [--batch]") @@ -99,5 +108,14 @@ for verify_cmd in [ rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'") print(out if rc == 0 else "runner2-removed") +print("Phase 5: warm images") +pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) +rc, out = ssh(f"set -e\n{pulls}") +if rc != 0: + print("Warmup failed") + print(out) + raise SystemExit(4) +print(out) + print("One-runner rollout complete for host") print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}") diff --git a/scripts/gitea-actions/runner-prechange-capture.xsh b/scripts/gitea-actions/runner-prechange-capture.xsh index 4609a54..527bd9b 100755 --- a/scripts/gitea-actions/runner-prechange-capture.xsh +++ b/scripts/gitea-actions/runner-prechange-capture.xsh @@ -37,6 +37,16 @@ def section(title, body): return f"\n{sep}\n{title}\n{sep}\n{body}\n" +def list_runner_containers(host): + rc, text = ssh( + host, + "docker ps -a --filter name=gitea-act-runner --format '{{.Names}}'", + ) + if rc != 0 or not text: + return [] + return [line.strip() for line in text.splitlines() if line.strip()] + + report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}" mkdir -p @(report_dir) report_path = os.path.join(report_dir, "capture.txt") @@ -57,39 +67,45 @@ for host, root in HOSTS.items(): rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'") data["ps"] = ps_text - rc1, inspect1 = ssh( - host, - "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", - ) - rc2, inspect2 = ssh( - host, - "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", - ) - if rc1 != 0: - inspect1 = "runner1-missing" - if rc2 != 0: - inspect2 = "runner2-missing" - data["inspect"] = inspect1 + "\n" + inspect2 + runner_containers = list_runner_containers(host) + data["runner_containers"] = runner_containers - rc, oom_scan = ssh( - host, - "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'", - ) - if rc != 0: - oom_scan = "oom-scan-unavailable" - data["oom_scan"] = oom_scan + inspect_lines = [] + oom_lines = [] + if not runner_containers: + inspect_lines.append("no-runner-containers-found") + oom_lines.append("no-runner-containers-found") + + for container in runner_containers: + rc, inspect_text = ssh( + host, + f"docker inspect {container} --format '{container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}'", + ) + if rc != 0: + inspect_text = f"{container}-inspect-unavailable" + inspect_lines.append(inspect_text) + + rc, oom_text = ssh( + host, + f"docker inspect {container} --format '{{{{.Name}}}} oom={{{{.State.OOMKilled}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}'", + ) + if rc != 0: + oom_text = f"{container}-oom-scan-unavailable" + oom_lines.append(oom_text) + + data["inspect"] = "\n".join(inspect_lines) + data["oom_scan"] = "\n".join(oom_lines) rc, mem = ssh(host, "free -h") data["mem"] = mem - rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1") - rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2") - if rc1 != 0: - log1 = "runner1-logs-unavailable" - if rc2 != 0: - log2 = "runner2-logs-unavailable" - data["log1"] = log1 - data["log2"] = log2 + runner_logs = {} + for container in runner_containers: + rc, log_text = ssh(host, f"docker logs --tail={TAIL_LINES} {container}") + if rc != 0: + log_text = f"{container}-logs-unavailable" + runner_logs[container] = log_text + data["runner_logs"] = runner_logs entries[host] = data @@ -108,8 +124,12 @@ with open(report_path, "w", encoding="utf-8") as handle: handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", ""))) handle.write(section("4. OOM SCAN", data.get("oom_scan", ""))) handle.write(section("5. MEMORY", data.get("mem", ""))) - handle.write(section("6. LOGS RUNNER1", data.get("log1", ""))) - handle.write(section("7. LOGS RUNNER2", data.get("log2", ""))) + containers = data.get("runner_containers", []) + if not containers: + handle.write(section("6. RUNNER LOGS", "no-runner-containers-found")) + else: + for idx, container in enumerate(containers, start=6): + handle.write(section(f"{idx}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, ""))) print("Capture complete") print(f"Report: {report_path}") -- 2.49.1 From 916b276c0827d97888214b8d866717681c192206 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 17:14:05 -0400 Subject: [PATCH 45/80] Fix E2E frontend path and Renovate endpoint --- .gitea/workflows/cicd.yaml | 2 +- .gitea/workflows/renovate.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index c1f4a39..12aa045 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2627,7 +2627,7 @@ jobs: -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \ -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ "${E2E_TESTER_DIGEST_REF}" \ - bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; cd /workspace/frontend; retry_cmd 5 4 yarn install --immutable; yarn test:e2e --reporter=list --timeout=90000' \ + bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \ 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 82dcafc..e2dd356 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -72,7 +72,7 @@ jobs: PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://dogar.darkhelm.org + RENOVATE_ENDPOINT: https://kankali.darkhelm.lan RENOVATE_ALLOW_INSECURE_TLS: "true" RENOVATE_GIT_AUTHOR: Renovate Bot RENOVATE_ONBOARDING: "false" -- 2.49.1 From f0c4b72dd6530db268cc0c2ad6bf2ae48fdb5be4 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 18:55:38 -0400 Subject: [PATCH 46/80] Pin registry auth realm host in build_cicd --- .gitea/workflows/cicd.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 12aa045..cdddd4d 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -194,6 +194,27 @@ jobs: return 1 } + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -373,6 +394,8 @@ jobs: docker system df || true fi + ensure_registry_auth_realm_host + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")" -- 2.49.1 From 7a3b6f5781a4cc1f97be0918189be2d1a868b3d0 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 18:58:48 -0400 Subject: [PATCH 47/80] Add guarded pre-job docker prune in CICD --- .gitea/workflows/cicd.yaml | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index cdddd4d..b6bf776 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -27,6 +27,8 @@ env: GITEA_REGISTRY: kankali.darkhelm.lan:3001 GITEA_REGISTRY_IP: 10.18.75.2 GITEA_REGISTRY_HOST: kankali.darkhelm.lan + CI_PRUNE_AT_JOB_START: "true" + CI_PRUNE_THRESHOLD_PERCENT: "90" defaults: run: @@ -57,6 +59,27 @@ jobs: echo "runner_arch=${RUNNER_ARCH:-unknown}" echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - name: Pre-job Docker prune + run: | + if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then + echo "Skipping pre-job prune" + exit 0 + fi + + used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" + echo "disk_used_before=${used_before}%" + echo "prune_threshold=${threshold}%" + + if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then + docker builder prune -af || true + docker system prune -af --volumes || true + used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + echo "disk_used_after=${used_after}%" + else + echo "Skipping prune; usage below threshold" + fi + - name: Resolve head SHA id: meta env: @@ -272,6 +295,27 @@ jobs: run: | echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - name: Pre-job Docker prune + run: | + if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then + echo "Skipping pre-job prune" + exit 0 + fi + + used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" + echo "disk_used_before=${used_before}%" + echo "prune_threshold=${threshold}%" + + if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then + docker builder prune -af || true + docker system prune -af --volumes || true + used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + echo "disk_used_after=${used_after}%" + else + echo "Skipping prune; usage below threshold" + fi + - name: Minimal checkout for CICD build inputs env: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} @@ -489,6 +533,21 @@ jobs: - &configure_registry_host_step name: Configure registry host resolution run: | + if [ "${CI_PRUNE_AT_JOB_START:-false}" = "true" ] && command -v docker >/dev/null 2>&1; then + used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" + echo "disk_used_before=${used_before}%" + echo "prune_threshold=${threshold}%" + if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then + docker builder prune -af || true + docker system prune -af --volumes || true + used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + echo "disk_used_after=${used_after}%" + else + echo "Skipping prune; usage below threshold" + fi + fi + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi -- 2.49.1 From c4d1102f6f36b05366356da520e5da381d2519a8 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 20:41:58 -0400 Subject: [PATCH 48/80] Fix build_cicd function scope and Renovate DNS fallback --- .gitea/workflows/cicd.yaml | 21 +++++++++++++++++++++ .gitea/workflows/renovate.yml | 8 ++++++++ 2 files changed, 29 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index b6bf776..31190b6 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -238,6 +238,27 @@ jobs: rm -f "${headers_file}" } + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index e2dd356..5d5a8e1 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -73,6 +73,7 @@ jobs: RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_PLATFORM: gitea RENOVATE_ENDPOINT: https://kankali.darkhelm.lan + RENOVATE_ENDPOINT_IP: 10.18.75.2 RENOVATE_ALLOW_INSECURE_TLS: "true" RENOVATE_GIT_AUTHOR: Renovate Bot RENOVATE_ONBOARDING: "false" @@ -101,6 +102,13 @@ jobs: ENDPOINT_HOST="$(printf '%s' "${RENOVATE_ENDPOINT_EFFECTIVE}" | sed -E 's#^https?://([^/]+).*$#\1#')" ENDPOINT_IP="$(getent hosts "${ENDPOINT_HOST}" | awk '{print $1; exit}' || true)" + if [ -z "${ENDPOINT_IP}" ] && [ -n "${RENOVATE_ENDPOINT_IP:-}" ]; then + ENDPOINT_IP="${RENOVATE_ENDPOINT_IP}" + echo "Using configured Renovate endpoint IP fallback: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" + if ! grep -q "${ENDPOINT_HOST}" /etc/hosts; then + echo "${ENDPOINT_IP} ${ENDPOINT_HOST}" >> /etc/hosts + fi + fi DOCKER_ADD_HOST_ARG="" if [ -n "${ENDPOINT_IP}" ]; then DOCKER_ADD_HOST_ARG="--add-host ${ENDPOINT_HOST}:${ENDPOINT_IP}" -- 2.49.1 From 955be4e2ec496ce70883ce27dae3148af90f704c Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 23:33:26 -0400 Subject: [PATCH 49/80] Fix missing registry auth helper in build_cicd job --- .gitea/workflows/cicd.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 31190b6..ace749f 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -448,6 +448,27 @@ jobs: return 1 } + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + echo "=== Pre-build disk telemetry ===" df -h || true if command -v docker >/dev/null 2>&1; then -- 2.49.1 From 47214027daa0b8373f0ce04694598c124c903144 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 23:37:04 -0400 Subject: [PATCH 50/80] Refactor CICD workflow anchors and remove duplicate helper --- .gitea/workflows/cicd.yaml | 189 +++++-------------------------------- 1 file changed, 22 insertions(+), 167 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index ace749f..9b4be03 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -47,7 +47,8 @@ jobs: base_hash: ${{ steps.base-state.outputs.base_hash }} head_sha: ${{ steps.meta.outputs.head_sha }} steps: - - name: Identify runner + - &identify_runner_step + name: Identify runner run: | echo "=== Runner Identity ===" echo "runner_name=${RUNNER_NAME:-}" @@ -59,7 +60,8 @@ jobs: echo "runner_arch=${RUNNER_ARCH:-unknown}" echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - name: Pre-job Docker prune + - &pre_job_prune_step + name: Pre-job Docker prune run: | if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then echo "Skipping pre-job prune" @@ -238,27 +240,6 @@ jobs: rm -f "${headers_file}" } - ensure_registry_auth_realm_host() { - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - headers_file="$(mktemp)" - if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then - realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" - if [ -n "${realm_url}" ]; then - realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" - if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then - if ! grep -q "${realm_host}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts - echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" - fi - fi - fi - fi - rm -f "${headers_file}" - } - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi @@ -316,26 +297,7 @@ jobs: run: | echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT" - - name: Pre-job Docker prune - run: | - if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then - echo "Skipping pre-job prune" - exit 0 - fi - - used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" - threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}" - echo "disk_used_before=${used_before}%" - echo "prune_threshold=${threshold}%" - - if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then - docker builder prune -af || true - docker system prune -af --volumes || true - used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" - echo "disk_used_after=${used_after}%" - else - echo "Skipping prune; usage below threshold" - fi + - *pre_job_prune_step - name: Minimal checkout for CICD build inputs env: @@ -560,17 +522,7 @@ jobs: timeout-minutes: 25 needs: build_cicd steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - &configure_registry_host_step name: Configure registry host resolution @@ -654,17 +606,7 @@ jobs: timeout-minutes: 30 needs: build_cicd steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - *configure_registry_host_step - *ensure_cicd_image_step @@ -700,17 +642,7 @@ jobs: timeout-minutes: 25 needs: build_cicd steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - *configure_registry_host_step - *ensure_cicd_image_step @@ -743,17 +675,7 @@ jobs: timeout-minutes: 15 needs: build_cicd steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - *configure_registry_host_step - *ensure_cicd_image_step @@ -816,7 +738,8 @@ jobs: backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }} backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }} steps: - - name: Resolve head SHA + - &resolve_head_sha_from_build_step + name: Resolve head SHA id: meta run: | echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" @@ -940,10 +863,7 @@ jobs: frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - *resolve_head_sha_from_build_step - name: Minimal checkout for frontend base image inputs env: @@ -1088,10 +1008,7 @@ jobs: integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - *resolve_head_sha_from_build_step - name: Minimal checkout for integration tester image inputs env: @@ -1213,10 +1130,7 @@ jobs: e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - *resolve_head_sha_from_build_step - name: Minimal checkout for E2E tester image inputs env: @@ -1380,22 +1294,9 @@ jobs: deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - *resolve_head_sha_from_build_step - name: Minimal checkout for runtime image inputs env: @@ -1609,10 +1510,7 @@ jobs: deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - *resolve_head_sha_from_build_step - name: Minimal checkout for backend main image inputs env: @@ -1745,10 +1643,7 @@ jobs: deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + - *resolve_head_sha_from_build_step - name: Minimal checkout for frontend main image inputs env: @@ -1909,17 +1804,7 @@ jobs: if: always() && (needs.runtime_images.result == 'failure' || needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') timeout-minutes: 10 steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - name: Collect runtime_images postmortem context env: @@ -2020,17 +1905,7 @@ jobs: if: always() && (needs['backend-tests'].result == 'failure' || needs['precommit-tests'].result == 'failure' || needs['frontend-tests'].result == 'failure' || needs.xdoctest.result == 'failure') timeout-minutes: 10 steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - name: Collect source lane postmortem context env: @@ -2125,17 +2000,7 @@ jobs: timeout-minutes: 20 needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image] steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - *configure_registry_host_step - name: Run runtime black-box integration checks via compose @@ -2409,17 +2274,7 @@ jobs: timeout-minutes: 45 needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-}" - echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" - echo "runner_hostname_env=${HOSTNAME:-unknown}" - echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" - echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - *configure_registry_host_step - name: Run E2E tests against runtime services via compose -- 2.49.1 From 383b88e87dd542d5fa52f5fab614ffe73445c987 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 07:48:06 -0400 Subject: [PATCH 51/80] Remove dead legacy runtime image workflow path --- .gitea/workflows/cicd.yaml | 269 ++++--------------------------------- 1 file changed, 24 insertions(+), 245 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 9b4be03..718063e 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -495,12 +495,7 @@ jobs: if: always() && needs.build_cicd.result == 'failure' timeout-minutes: 10 steps: - - name: Identify runner - run: | - echo "=== Runner Identity ===" - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + - *identify_runner_step - name: Build CICD postmortem diagnostics run: | @@ -549,7 +544,7 @@ jobs: - &ensure_cicd_image_step name: Ensure CICD image is available env: - HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || needs.runtime_images.outputs.head_sha || github.sha }} + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || github.sha }} run: | IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" if docker image inspect "${IMAGE}" >/dev/null 2>&1; then @@ -1281,226 +1276,6 @@ jobs: - *failure_diagnostics_step - runtime_images: - name: Build and Publish Runtime Images (Legacy Disabled) - runs-on: ubuntu-act-8gb - if: false - needs: [build_cicd, cicd-tests-complete, build-backend-base-image, build-frontend-base-image] - timeout-minutes: 45 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} - deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} - deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} - steps: - - *identify_runner_step - - - *resolve_head_sha_from_build_step - - - name: Minimal checkout for runtime image inputs - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - umask 077 - trap 'rm -f ~/.ssh/id_rsa' EXIT - - retry_cmd() { - attempts="${1:-5}" - backoff="${2:-2}" - shift 2 - - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if "$@"; then - return 0 - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - echo "Command failed after ${attempts} attempts: $*" - return 1 - } - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - mkdir -p ~/.ssh - echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . - - if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- \ - .dockerignore \ - Dockerfile.backend \ - Dockerfile.frontend \ - backend \ - frontend \ - scripts/check-dockerfile-boundaries.sh \ - scripts/verify-deployable-image-purity.sh - else - git checkout HEAD -- \ - .dockerignore \ - Dockerfile.backend \ - Dockerfile.frontend \ - backend \ - frontend \ - scripts/check-dockerfile-boundaries.sh \ - scripts/verify-deployable-image-purity.sh - fi - - - name: Verify deployable runtime boundaries - run: | - set -euo pipefail - bash ./scripts/check-dockerfile-boundaries.sh - - - name: Build and verify deployable runtime image purity - env: - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" . - docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" . - - bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend - bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend - - - name: Push deployable backend runtime image - id: deployable_backend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - retry_registry_op() { - op_name="$1" - image_ref="$2" - attempts="${3:-5}" - backoff="${4:-3}" - attempt=1 - - while [ "${attempt}" -le "${attempts}" ]; do - echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" - if [ "${op_name}" = "push" ]; then - if docker push "${image_ref}"; then - return 0 - fi - else - if docker pull "${image_ref}" >/dev/null; then - return 0 - fi - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - echo "${op_name} failed. Probing registry endpoint before retry..." - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true - sleep_seconds=$((backoff * attempt)) - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - return 1 - } - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" - DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" - - retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 - retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 - - DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" - - if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve backend digest after push/pull retries" - exit 1 - fi - - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - - name: Push deployable frontend runtime image - id: deployable_frontend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - retry_registry_op() { - op_name="$1" - image_ref="$2" - attempts="${3:-5}" - backoff="${4:-3}" - attempt=1 - - while [ "${attempt}" -le "${attempts}" ]; do - echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" - if [ "${op_name}" = "push" ]; then - if docker push "${image_ref}"; then - return 0 - fi - else - if docker pull "${image_ref}" >/dev/null; then - return 0 - fi - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - echo "${op_name} failed. Probing registry endpoint before retry..." - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true - sleep_seconds=$((backoff * attempt)) - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - return 1 - } - - echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" - DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" - docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" - - retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 - retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 - - DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" - - if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve frontend digest after push/pull retries" - exit 1 - fi - - echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - - *failure_diagnostics_step - build-backend-main-image: name: Build Backend Main Image runs-on: ubuntu-act-8gb @@ -1796,17 +1571,17 @@ jobs: - *failure_diagnostics_step runtime-images-postmortem: - name: Runtime Images Failure Postmortem + name: Production Image Failures Postmortem # Run on the broader runner pool so we can still capture diagnostics when - # runtime_images fails during platform "Set up job" before any step executes. + # one of the active production image jobs fails during platform setup. runs-on: ubuntu-act - needs: [runtime_images, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] - if: always() && (needs.runtime_images.result == 'failure' || needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') + needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + if: always() && (needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') timeout-minutes: 10 steps: - *identify_runner_step - - name: Collect runtime_images postmortem context + - name: Collect production image postmortem context env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -1819,7 +1594,10 @@ jobs: echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" echo "repository=${GITHUB_REPOSITORY:-unknown}" echo "server_url=${GITHUB_SERVER_URL:-unknown}" - echo "job_result_runtime_images=${{ needs.runtime_images.result }}" + echo "build_backend_main_image_result=${{ needs['build-backend-main-image'].result }}" + echo "build_frontend_main_image_result=${{ needs['build-frontend-main-image'].result }}" + echo "build_integration_tester_image_result=${{ needs['build-integration-tester-image'].result }}" + echo "build_e2e_tester_image_result=${{ needs['build-e2e-tester-image'].result }}" echo "=== Local Runner Telemetry (postmortem job host) ===" uname -a || true @@ -1875,20 +1653,21 @@ jobs: ' completed = _g(job, "completed_at", "end_time")' \ ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \ '' \ - 'target = None' \ + 'targets = {' \ + ' "Build Backend Main Image",' \ + ' "Build Frontend Main Image",' \ + ' "Build Integration Tester Image",' \ + ' "Build E2E Tester Image",' \ + '}' \ + '' \ + 'print("=== targeted production image jobs ===")' \ 'for job in jobs:' \ ' name = str(_g(job, "name", "job_name"))' \ - ' if "Build and Publish Runtime Images" in name:' \ - ' target = job' \ - ' break' \ - '' \ - 'if target is None:' \ - ' print("runtime_images_job=not_found")' \ - 'else:' \ - ' print("=== runtime_images job detail ===")' \ - ' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \ - ' if key in target:' \ - ' print(f"{key}={target[key]}")')" + ' if name in targets:' \ + ' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \ + ' if key in job:' \ + ' print(f"{key}={job[key]}")' \ + ' print("---")')" else echo "python3 unavailable; emitting raw jobs payload head" sed -n '1,120p' /tmp/actions-jobs.json || true -- 2.49.1 From a48c7e6ba43bcfa8f747733f5ce93f7442cf888f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 07:54:51 -0400 Subject: [PATCH 52/80] Fix Renovate internal and external Gitea endpoint selection --- .gitea/workflows/renovate.yml | 116 +++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 37 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 5d5a8e1..8c0163b 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -72,7 +72,8 @@ jobs: PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://kankali.darkhelm.lan + RENOVATE_ENDPOINT_INTERNAL: http://kankali.darkhelm.lan:3001/api/v1 + RENOVATE_ENDPOINT_EXTERNAL: https://dogar.darkhelm.org/api/v1 RENOVATE_ENDPOINT_IP: 10.18.75.2 RENOVATE_ALLOW_INSECURE_TLS: "true" RENOVATE_GIT_AUTHOR: Renovate Bot @@ -86,37 +87,6 @@ jobs: TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" - RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" - if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then - API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" - BASE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" - else - API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" - BASE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" - fi - - export RENOVATE_ENDPOINT="${BASE_ENDPOINT}" - echo "Renovate endpoint (primary): ${RENOVATE_ENDPOINT}" - echo "Renovate endpoint (fallback): ${API_ENDPOINT}" - echo "Preflight API endpoint: ${API_ENDPOINT}" - - ENDPOINT_HOST="$(printf '%s' "${RENOVATE_ENDPOINT_EFFECTIVE}" | sed -E 's#^https?://([^/]+).*$#\1#')" - ENDPOINT_IP="$(getent hosts "${ENDPOINT_HOST}" | awk '{print $1; exit}' || true)" - if [ -z "${ENDPOINT_IP}" ] && [ -n "${RENOVATE_ENDPOINT_IP:-}" ]; then - ENDPOINT_IP="${RENOVATE_ENDPOINT_IP}" - echo "Using configured Renovate endpoint IP fallback: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" - if ! grep -q "${ENDPOINT_HOST}" /etc/hosts; then - echo "${ENDPOINT_IP} ${ENDPOINT_HOST}" >> /etc/hosts - fi - fi - DOCKER_ADD_HOST_ARG="" - if [ -n "${ENDPOINT_IP}" ]; then - DOCKER_ADD_HOST_ARG="--add-host ${ENDPOINT_HOST}:${ENDPOINT_IP}" - echo "Renovate container host mapping: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" - else - echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host" - fi - CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then echo "⚠ Renovate insecure TLS mode enabled for self-signed certificate endpoint" @@ -124,9 +94,79 @@ jobs: export NODE_TLS_REJECT_UNAUTHORIZED=0 fi - VERSION_STATUS=$(curl -sS -o /tmp/renovate-version-check.json -w "%{http_code}" \ - ${CURL_INSECURE_FLAG} \ - "${API_ENDPOINT}/version" || true) + configure_endpoint() { + endpoint_input="${1%/}" + endpoint_ip_fallback="${2:-}" + + if [[ "${endpoint_input}" == */api/v1 ]]; then + API_ENDPOINT="${endpoint_input}" + BASE_ENDPOINT="${endpoint_input%/api/v1}" + else + API_ENDPOINT="${endpoint_input}/api/v1" + BASE_ENDPOINT="${endpoint_input}" + fi + + ENDPOINT_AUTHORITY="$(printf '%s' "${BASE_ENDPOINT}" | sed -E 's#^https?://([^/]+).*$#\1#')" + ENDPOINT_HOST="${ENDPOINT_AUTHORITY%%:*}" + ENDPOINT_IP="$(getent hosts "${ENDPOINT_HOST}" | awk '{print $1; exit}' || true)" + if [ -z "${ENDPOINT_IP}" ] && [ -n "${endpoint_ip_fallback}" ]; then + ENDPOINT_IP="${endpoint_ip_fallback}" + echo "Using configured Renovate endpoint IP fallback: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" + if ! grep -q "${ENDPOINT_HOST}" /etc/hosts; then + echo "${ENDPOINT_IP} ${ENDPOINT_HOST}" >> /etc/hosts + fi + fi + + DOCKER_ADD_HOST_ARG="" + if [ -n "${ENDPOINT_IP}" ]; then + DOCKER_ADD_HOST_ARG="--add-host ${ENDPOINT_HOST}:${ENDPOINT_IP}" + echo "Renovate container host mapping: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}" + else + echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host" + fi + } + + select_reachable_endpoint() { + for candidate in "${RENOVATE_ENDPOINT_INTERNAL}" "${RENOVATE_ENDPOINT_EXTERNAL}"; do + fallback_ip="" + if [ "${candidate}" = "${RENOVATE_ENDPOINT_INTERNAL}" ]; then + fallback_ip="${RENOVATE_ENDPOINT_IP:-}" + fi + + configure_endpoint "${candidate}" "${fallback_ip}" + echo "Testing Renovate endpoint candidate: ${API_ENDPOINT}" + + version_status=$(curl -sS -o /tmp/renovate-version-check.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + "${API_ENDPOINT}/version" || true) + + if [ "${version_status}" = "200" ]; then + SELECTED_ENDPOINT_INPUT="${candidate}" + VERSION_STATUS="${version_status}" + return 0 + fi + + echo "⚠ Endpoint candidate failed version preflight: ${API_ENDPOINT} (status=${version_status})" + if [ -s /tmp/renovate-version-check.json ]; then + cat /tmp/renovate-version-check.json || true + fi + done + + return 1 + } + + if ! select_reachable_endpoint; then + echo "❌ Unable to reach either configured Renovate endpoint" + echo "Tried internal: ${RENOVATE_ENDPOINT_INTERNAL}" + echo "Tried external: ${RENOVATE_ENDPOINT_EXTERNAL}" + exit 1 + fi + + export RENOVATE_ENDPOINT="${BASE_ENDPOINT}" + echo "Renovate endpoint (primary): ${RENOVATE_ENDPOINT}" + echo "Renovate endpoint (fallback): ${RENOVATE_ENDPOINT_EXTERNAL}" + echo "Preflight API endpoint: ${API_ENDPOINT}" + PLATFORM_VERSION="" if [ "${VERSION_STATUS}" = "200" ]; then PLATFORM_VERSION=$(sed -n 's/.*"version":"\([^"]*\)".*/\1/p' /tmp/renovate-version-check.json | head -n 1) @@ -139,7 +179,9 @@ jobs: fi else echo "⚠ Gitea version preflight failed with status ${VERSION_STATUS}" - cat /tmp/renovate-version-check.json || true + if [ -s /tmp/renovate-version-check.json ]; then + cat /tmp/renovate-version-check.json || true + fi fi select_token_with_repo_access() { @@ -197,7 +239,7 @@ jobs: SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" - export RENOVATE_HOST_RULES="[{\"matchHost\":\"${BASE_ENDPOINT}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" + export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" -- 2.49.1 From d0802f70c9f92365ac46513328767ae34ef61f02 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 09:09:26 -0400 Subject: [PATCH 53/80] Relax Renovate token preflight and fix E2E frontend mount --- .gitea/workflows/cicd.yaml | 2 +- .gitea/workflows/renovate.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 718063e..16f54d1 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2382,7 +2382,7 @@ jobs: -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ - -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \ + -v "${E2E_REPO_DIR}:/workspace" \ -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ "${E2E_TESTER_DIGEST_REF}" \ bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \ diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 8c0163b..9f92f7d 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -206,7 +206,10 @@ jobs: -H "Authorization: token ${candidate_value}" \ "${API_ENDPOINT}/orgs/${TARGET_ORG}" || true) - if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then + if [ "${REPO_STATUS}" = "200" ]; then + if [ "${USER_STATUS}" != "200" ] || [ "${ORG_STATUS}" != "200" ]; then + echo "⚠ Token candidate ${candidate_name} has repo access but limited user/org scopes (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})" + fi echo "${candidate_name}:${candidate_value}" return 0 fi @@ -219,7 +222,7 @@ jobs: if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then echo "❌ No token available for Renovate authentication with repository access" - echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." + echo "Configure RENOVATE_TOKEN with at least repository read/write access; user/org read scopes are optional for this workflow." echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}" if [ -s /tmp/renovate-auth-check-user.json ]; then echo "Last user endpoint response body:" -- 2.49.1 From 85960e3599b2d09aeddb38702bb9efbe1a4f16aa Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 09:42:06 -0400 Subject: [PATCH 54/80] Avoid Renovate preflight failure on missing read:user scope --- .gitea/workflows/renovate.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 9f92f7d..70e440c 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -253,10 +253,11 @@ jobs: # shellcheck disable=SC2086 docker run --rm ${DOCKER_ADD_HOST_ARG} \ -e API_ENDPOINT="${API_ENDPOINT}" \ + -e TARGET_REPO="${TARGET_REPO}" \ -e RENOVATE_TOKEN \ -e NODE_TLS_REJECT_UNAUTHORIZED \ "${RENOVATE_IMAGE}" \ - node -e 'const endpoints = [["version", `${process.env.API_ENDPOINT}/version`, {}], ["user", `${process.env.API_ENDPOINT}/user`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` } }]]; let failed = false; (async () => { for (const [name, url, options] of endpoints) { try { const response = await fetch(url, options); console.log(`container_${name}_status=${response.status}`); if (!response.ok) { failed = true; console.log(`container_${name}_body_start`); console.log((await response.text()).slice(0, 1000)); console.log(`container_${name}_body_end`); } } catch (error) { failed = true; console.log(`container_${name}_error=${error instanceof Error ? error.message : String(error)}`); } } if (failed) { process.exit(1); } })();' + node -e 'const endpoints = [["version", `${process.env.API_ENDPOINT}/version`, {}], ["repo", `${process.env.API_ENDPOINT}/repos/${process.env.TARGET_REPO}`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` } }], ["user", `${process.env.API_ENDPOINT}/user`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` }, optional: true }]]; let failed = false; (async () => { for (const [name, url, rawOptions] of endpoints) { const optional = Boolean(rawOptions.optional); const options = { ...rawOptions }; delete options.optional; try { const response = await fetch(url, options); console.log(`container_${name}_status=${response.status}`); if (!response.ok && !optional) { failed = true; console.log(`container_${name}_body_start`); console.log((await response.text()).slice(0, 1000)); console.log(`container_${name}_body_end`); } else if (!response.ok && optional) { console.log(`container_${name}_optional_scope_missing=true`); } } catch (error) { if (!optional) { failed = true; console.log(`container_${name}_error=${error instanceof Error ? error.message : String(error)}`); } else { console.log(`container_${name}_optional_probe_error=${error instanceof Error ? error.message : String(error)}`); } } } if (failed) { process.exit(1); } })();' # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then -- 2.49.1 From 331fb887d5a33b0f6eae840c5daea1ec19b3e7c4 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 10:15:12 -0400 Subject: [PATCH 55/80] Fix Renovate repo parsing and git TLS clone in runner --- .gitea/workflows/renovate.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 70e440c..eaa71e9 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -276,6 +276,8 @@ jobs: docker run --rm ${DOCKER_ADD_HOST_ARG} \ -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ + -e GIT_SSL_NO_VERIFY=1 \ + -e RENOVATE_GIT_NO_VERIFY=true \ -e RENOVATE_TOKEN \ -e RENOVATE_HOST_RULES \ -e RENOVATE_X_PLATFORM_VERSION \ @@ -292,7 +294,6 @@ jobs: --endpoint "${RENOVATE_ENDPOINT}" \ --git-author "${RENOVATE_GIT_AUTHOR}" \ --onboarding="${RENOVATE_ONBOARDING}" \ - --require-config "${RENOVATE_REQUIRE_CONFIG}" \ --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" \ --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" \ DarkHelm.org/plex-playlist -- 2.49.1 From a3e864cfcd3131ccedab27ff3a0cea1e2a9a6f88 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 10:30:30 -0400 Subject: [PATCH 56/80] Remove invalid Renovate gitNoVerify env override --- .gitea/workflows/renovate.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index eaa71e9..188d00c 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -277,7 +277,6 @@ jobs: -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e GIT_SSL_NO_VERIFY=1 \ - -e RENOVATE_GIT_NO_VERIFY=true \ -e RENOVATE_TOKEN \ -e RENOVATE_HOST_RULES \ -e RENOVATE_X_PLATFORM_VERSION \ -- 2.49.1 From 0ac17105b846e44e1b845c537e22deb6d06a68c2 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 10:52:39 -0400 Subject: [PATCH 57/80] Add explicit git host credentials for Renovate clone --- .gitea/workflows/renovate.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 188d00c..e9f6af0 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -74,6 +74,8 @@ jobs: RENOVATE_PLATFORM: gitea RENOVATE_ENDPOINT_INTERNAL: http://kankali.darkhelm.lan:3001/api/v1 RENOVATE_ENDPOINT_EXTERNAL: https://dogar.darkhelm.org/api/v1 + RENOVATE_GIT_HOST: dogar.darkhelm.org + RENOVATE_GIT_USERNAME: ${{ github.actor }} RENOVATE_ENDPOINT_IP: 10.18.75.2 RENOVATE_ALLOW_INSECURE_TLS: "true" RENOVATE_GIT_AUTHOR: Renovate Bot @@ -242,7 +244,7 @@ jobs: SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" - export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" + export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"},{\"matchHost\":\"${RENOVATE_GIT_HOST}\",\"username\":\"${RENOVATE_GIT_USERNAME}\",\"password\":\"${SELECTED_TOKEN}\",\"authType\":\"Basic\"}]" unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" -- 2.49.1 From 9a68fa11096874d89163a4a8c34be211cc466d56 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 16:36:31 -0400 Subject: [PATCH 58/80] Finalize workflow fixes for Renovate and E2E --- .gitea/workflows/cicd.yaml | 26 ++++++++++++++++---------- .gitea/workflows/renovate.yml | 12 ++++++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 16f54d1..c6283c2 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -2258,6 +2258,12 @@ jobs: git -C "${E2E_REPO_DIR}" checkout HEAD -- frontend fi + if [ ! -f "${E2E_REPO_DIR}/frontend/package.json" ]; then + echo "❌ Missing frontend/package.json in cloned E2E workspace" + find "${E2E_REPO_DIR}" -maxdepth 3 -type f | sort || true + exit 1 + fi + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3 @@ -2376,16 +2382,16 @@ jobs: exit 1 fi - docker run --rm \ - --network "${COMPOSE_PROJECT_NAME}_default" \ - -e CI=true \ - -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ - -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ - -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ - -v "${E2E_REPO_DIR}:/workspace" \ - -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ - "${E2E_TESTER_DIGEST_REF}" \ - bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \ + tar -C "${E2E_REPO_DIR}" -cf - . \ + | docker run --rm -i \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + -e CI=true \ + -e PLAYWRIGHT_BASE_URL="http://frontend:80" \ + -e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \ + -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ + -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ + "${E2E_TESTER_DIGEST_REF}" \ + bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; mkdir -p /workspace; tar -xf - -C /workspace; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; find /workspace -maxdepth 3 -type f | sort | head -n 100; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \ 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index e9f6af0..730638f 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -76,6 +76,7 @@ jobs: RENOVATE_ENDPOINT_EXTERNAL: https://dogar.darkhelm.org/api/v1 RENOVATE_GIT_HOST: dogar.darkhelm.org RENOVATE_GIT_USERNAME: ${{ github.actor }} + RENOVATE_GIT_HOST_IP: 10.18.75.2 RENOVATE_ENDPOINT_IP: 10.18.75.2 RENOVATE_ALLOW_INSECURE_TLS: "true" RENOVATE_GIT_AUTHOR: Renovate Bot @@ -126,6 +127,17 @@ jobs: else echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host" fi + + # Keep Dogar as canonical git host, but pin it to the internal LAN IP from the runner. + GIT_HOST_IP="${RENOVATE_GIT_HOST_IP:-}" + if [ -z "${GIT_HOST_IP}" ]; then + GIT_HOST_IP="$(getent hosts "${RENOVATE_GIT_HOST}" | awk '{print $1; exit}' || true)" + fi + + if [ -n "${GIT_HOST_IP}" ] && [ "${RENOVATE_GIT_HOST}" != "${ENDPOINT_HOST}" ]; then + DOCKER_ADD_HOST_ARG="${DOCKER_ADD_HOST_ARG} --add-host ${RENOVATE_GIT_HOST}:${GIT_HOST_IP}" + echo "Renovate container git host mapping: ${RENOVATE_GIT_HOST} -> ${GIT_HOST_IP}" + fi } select_reachable_endpoint() { -- 2.49.1 From 64bd56e36e9d4b9bb0941a1fb563ccc5d95b1640 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 16:50:01 -0400 Subject: [PATCH 59/80] Force Renovate git URL to selected endpoint base --- .gitea/workflows/renovate.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 730638f..3ee33fa 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -176,9 +176,13 @@ jobs: exit 1 fi + # Force git clone base URL to the selected endpoint host/port. + # This avoids relying on platform-advertised clone URLs that may point to an unreachable external proxy. + export RENOVATE_GIT_URL="${BASE_ENDPOINT}/" export RENOVATE_ENDPOINT="${BASE_ENDPOINT}" echo "Renovate endpoint (primary): ${RENOVATE_ENDPOINT}" echo "Renovate endpoint (fallback): ${RENOVATE_ENDPOINT_EXTERNAL}" + echo "Renovate git base URL override: ${RENOVATE_GIT_URL}" echo "Preflight API endpoint: ${API_ENDPOINT}" PLATFORM_VERSION="" @@ -292,6 +296,7 @@ jobs: -e NODE_TLS_REJECT_UNAUTHORIZED \ -e GIT_SSL_NO_VERIFY=1 \ -e RENOVATE_TOKEN \ + -e RENOVATE_GIT_URL \ -e RENOVATE_HOST_RULES \ -e RENOVATE_X_PLATFORM_VERSION \ -e RENOVATE_PLATFORM \ @@ -305,6 +310,7 @@ jobs: "${RENOVATE_IMAGE}" \ --platform "${RENOVATE_PLATFORM}" \ --endpoint "${RENOVATE_ENDPOINT}" \ + --git-url "${RENOVATE_GIT_URL}" \ --git-author "${RENOVATE_GIT_AUTHOR}" \ --onboarding="${RENOVATE_ONBOARDING}" \ --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" \ -- 2.49.1 From 72e775d2b73ce3db6807e929288d425e90035b23 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 17:47:01 -0400 Subject: [PATCH 60/80] Remove incorrect dogar host IP override for Renovate --- .gitea/workflows/renovate.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 3ee33fa..5b1bb80 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -76,7 +76,6 @@ jobs: RENOVATE_ENDPOINT_EXTERNAL: https://dogar.darkhelm.org/api/v1 RENOVATE_GIT_HOST: dogar.darkhelm.org RENOVATE_GIT_USERNAME: ${{ github.actor }} - RENOVATE_GIT_HOST_IP: 10.18.75.2 RENOVATE_ENDPOINT_IP: 10.18.75.2 RENOVATE_ALLOW_INSECURE_TLS: "true" RENOVATE_GIT_AUTHOR: Renovate Bot @@ -128,16 +127,6 @@ jobs: echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host" fi - # Keep Dogar as canonical git host, but pin it to the internal LAN IP from the runner. - GIT_HOST_IP="${RENOVATE_GIT_HOST_IP:-}" - if [ -z "${GIT_HOST_IP}" ]; then - GIT_HOST_IP="$(getent hosts "${RENOVATE_GIT_HOST}" | awk '{print $1; exit}' || true)" - fi - - if [ -n "${GIT_HOST_IP}" ] && [ "${RENOVATE_GIT_HOST}" != "${ENDPOINT_HOST}" ]; then - DOCKER_ADD_HOST_ARG="${DOCKER_ADD_HOST_ARG} --add-host ${RENOVATE_GIT_HOST}:${GIT_HOST_IP}" - echo "Renovate container git host mapping: ${RENOVATE_GIT_HOST} -> ${GIT_HOST_IP}" - fi } select_reachable_endpoint() { -- 2.49.1 From fa0f58275981a5199593da4024603cfa8969de9a Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 18:05:03 -0400 Subject: [PATCH 61/80] Rewrite Renovate git clone URLs to internal authenticated host --- .gitea/workflows/renovate.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 5b1bb80..ba13ac7 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -250,6 +250,12 @@ jobs: SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"},{\"matchHost\":\"${RENOVATE_GIT_HOST}\",\"username\":\"${RENOVATE_GIT_USERNAME}\",\"password\":\"${SELECTED_TOKEN}\",\"authType\":\"Basic\"}]" + # Rewrite external clone URLs to an authenticated internal URL for git operations. + export RENOVATE_GIT_REWRITE_FROM="https://dogar.darkhelm.org/" + export RENOVATE_GIT_REWRITE_TO="http://${RENOVATE_GIT_USERNAME}:${SELECTED_TOKEN}@kankali.darkhelm.lan:3001/" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0="url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" + export GIT_CONFIG_VALUE_0="${RENOVATE_GIT_REWRITE_FROM}" unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" @@ -284,6 +290,9 @@ jobs: -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e GIT_SSL_NO_VERIFY=1 \ + -e GIT_CONFIG_COUNT \ + -e GIT_CONFIG_KEY_0 \ + -e GIT_CONFIG_VALUE_0 \ -e RENOVATE_TOKEN \ -e RENOVATE_GIT_URL \ -e RENOVATE_HOST_RULES \ -- 2.49.1 From 9bb36d18a4792d07a183550ad1a3a17d26b045d8 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 18:57:57 -0400 Subject: [PATCH 62/80] Configure in-container git rewrite for Renovate clone auth --- .gitea/workflows/renovate.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index ba13ac7..cb379e0 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -249,13 +249,10 @@ jobs: SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" - export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"},{\"matchHost\":\"${RENOVATE_GIT_HOST}\",\"username\":\"${RENOVATE_GIT_USERNAME}\",\"password\":\"${SELECTED_TOKEN}\",\"authType\":\"Basic\"}]" + export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" # Rewrite external clone URLs to an authenticated internal URL for git operations. export RENOVATE_GIT_REWRITE_FROM="https://dogar.darkhelm.org/" export RENOVATE_GIT_REWRITE_TO="http://${RENOVATE_GIT_USERNAME}:${SELECTED_TOKEN}@kankali.darkhelm.lan:3001/" - export GIT_CONFIG_COUNT=1 - export GIT_CONFIG_KEY_0="url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" - export GIT_CONFIG_VALUE_0="${RENOVATE_GIT_REWRITE_FROM}" unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" @@ -290,10 +287,10 @@ jobs: -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e GIT_SSL_NO_VERIFY=1 \ - -e GIT_CONFIG_COUNT \ - -e GIT_CONFIG_KEY_0 \ - -e GIT_CONFIG_VALUE_0 \ -e RENOVATE_TOKEN \ + -e RENOVATE_GIT_USERNAME \ + -e RENOVATE_GIT_REWRITE_FROM \ + -e RENOVATE_GIT_REWRITE_TO \ -e RENOVATE_GIT_URL \ -e RENOVATE_HOST_RULES \ -e RENOVATE_X_PLATFORM_VERSION \ @@ -305,15 +302,9 @@ jobs: -e RENOVATE_REQUIRE_CONFIG \ -e RENOVATE_PR_CONCURRENT_LIMIT \ -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ + --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ - --platform "${RENOVATE_PLATFORM}" \ - --endpoint "${RENOVATE_ENDPOINT}" \ - --git-url "${RENOVATE_GIT_URL}" \ - --git-author "${RENOVATE_GIT_AUTHOR}" \ - --onboarding="${RENOVATE_ONBOARDING}" \ - --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" \ - --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" \ - DarkHelm.org/plex-playlist + -lc 'set -e; git config --global "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}"; renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" DarkHelm.org/plex-playlist' } if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then -- 2.49.1 From 6aadca5df2a07d36a67faaa2ebbe7dfc5e78251f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 10 Jul 2026 23:06:22 -0400 Subject: [PATCH 63/80] Prefer local kankali:2222 SSH clone with HTTP fallback --- .gitea/workflows/renovate.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index cb379e0..40c898e 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -250,9 +250,20 @@ jobs: SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" export RENOVATE_TOKEN="${SELECTED_TOKEN}" export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]" - # Rewrite external clone URLs to an authenticated internal URL for git operations. + # Rewrite external clone URLs to local-network git transport. export RENOVATE_GIT_REWRITE_FROM="https://dogar.darkhelm.org/" export RENOVATE_GIT_REWRITE_TO="http://${RENOVATE_GIT_USERNAME}:${SELECTED_TOKEN}@kankali.darkhelm.lan:3001/" + + # Prefer local SSH for git on kankali:2222 when keys are available in the container. + if docker run --rm ${DOCKER_ADD_HOST_ARG} \ + --entrypoint /bin/sh \ + "${RENOVATE_IMAGE}" \ + -lc 'GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5" git ls-remote --heads "ssh://git@kankali.darkhelm.lan:2222/'"${TARGET_REPO}"'.git" >/dev/null 2>&1'; then + export RENOVATE_GIT_REWRITE_TO="ssh://git@kankali.darkhelm.lan:2222/" + echo "✓ Renovate git rewrite transport: local SSH (kankali:2222)" + else + echo "⚠ Local SSH git auth unavailable in Renovate container; using internal HTTP token fallback" + fi unset SELECTED_TOKEN RESULT_TOKEN echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" @@ -287,6 +298,7 @@ jobs: -e LOG_LEVEL \ -e NODE_TLS_REJECT_UNAUTHORIZED \ -e GIT_SSL_NO_VERIFY=1 \ + -e GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no' \ -e RENOVATE_TOKEN \ -e RENOVATE_GIT_USERNAME \ -e RENOVATE_GIT_REWRITE_FROM \ -- 2.49.1 From 4ebf9ef214534d1f2bbba71a177950670123f819 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 08:36:21 -0400 Subject: [PATCH 64/80] Keep CI git/registry traffic on local kankali endpoints --- .gitea/workflows/cicd.yaml | 12 ++++++------ .gitea/workflows/renovate.yml | 10 +++++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index c6283c2..c32272c 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -509,7 +509,7 @@ jobs: docker version || true docker info || true fi - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true backend-tests: name: Backend Tests @@ -1205,7 +1205,7 @@ jobs: echo "${GITEA_REGISTRY_HOST} DNS resolution: failed" fi - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" >/dev/null || true if docker pull "${image_ref}"; then return 0 @@ -1610,7 +1610,7 @@ jobs: fi echo "=== Registry Reachability Probe ===" - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true if [ -z "${GITHUB_TOKEN:-}" ]; then echo "No GITHUB_TOKEN available for API diagnostics" @@ -1715,7 +1715,7 @@ jobs: fi echo "=== Registry Reachability Probe ===" - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true if [ -z "${GITHUB_TOKEN:-}" ]; then echo "No GITHUB_TOKEN available for API diagnostics" @@ -2044,7 +2044,7 @@ jobs: docker version || true docker info || true fi - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true e2e-tests: name: End-to-End Tests @@ -2429,4 +2429,4 @@ jobs: docker version || true docker info || true fi - timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true + timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 40c898e..742ef7f 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -316,7 +316,15 @@ jobs: -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ - -lc 'set -e; git config --global "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}"; renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" DarkHelm.org/plex-playlist' + -lc 'set -e + git config --global "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "http://dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://${RENOVATE_GIT_USERNAME}@dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://${RENOVATE_TOKEN}@dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/" + git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}" + echo "Configured git dogar->kankali rewrite rules" + renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" DarkHelm.org/plex-playlist' } if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then -- 2.49.1 From fb0d7dabec5854d88f201ed1170db6939544c283 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 08:45:30 -0400 Subject: [PATCH 65/80] Use local renovate image mirror before GHCR fallback --- .gitea/workflows/renovate.yml | 41 ++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 742ef7f..1b4a34f 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -22,7 +22,10 @@ jobs: steps: - name: Prepare Renovate container image env: - RENOVATE_IMAGE: ghcr.io/renovatebot/renovate:41 + RENOVATE_IMAGE_PRIMARY: kankali.darkhelm.lan:3001/darkhelm.org/renovate:41 + RENOVATE_IMAGE_FALLBACK: ghcr.io/renovatebot/renovate:41 + GITEA_REGISTRY_HOST: kankali.darkhelm.lan + GITEA_REGISTRY_IP: 10.18.75.2 run: | set -euo pipefail @@ -51,10 +54,42 @@ jobs: } echo "=== Preparing Renovate container ===" - echo "Using Renovate image: ${RENOVATE_IMAGE}" free -h || true - retry_cmd 3 15 docker pull "${RENOVATE_IMAGE}" + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + pick_renovate_image() { + for candidate in "${RENOVATE_IMAGE_PRIMARY}" "${RENOVATE_IMAGE_FALLBACK}"; do + echo "Trying Renovate image candidate: ${candidate}" + + if docker image inspect "${candidate}" >/dev/null 2>&1; then + echo "Using cached Renovate image: ${candidate}" + echo "${candidate}" + return 0 + fi + + if retry_cmd 3 15 docker pull "${candidate}"; then + echo "${candidate}" + return 0 + fi + + echo "⚠ Renovate image candidate failed: ${candidate}" + done + + return 1 + } + + if ! RENOVATE_IMAGE="$(pick_renovate_image)"; then + echo "❌ Unable to prepare Renovate image from local mirror or GHCR" + echo "Tried: ${RENOVATE_IMAGE_PRIMARY}" + echo "Tried: ${RENOVATE_IMAGE_FALLBACK}" + exit 1 + fi + + echo "Using Renovate image: ${RENOVATE_IMAGE}" + docker run --rm "${RENOVATE_IMAGE}" --version echo "RENOVATE_IMAGE=${RENOVATE_IMAGE}" >> "$GITHUB_ENV" -- 2.49.1 From 8a32017242b0fb658b92ae5ddb84b45abb310710 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 09:06:47 -0400 Subject: [PATCH 66/80] Fix renovate image selector output for docker run --- .gitea/workflows/renovate.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 1b4a34f..d0fa22a 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -62,20 +62,20 @@ jobs: pick_renovate_image() { for candidate in "${RENOVATE_IMAGE_PRIMARY}" "${RENOVATE_IMAGE_FALLBACK}"; do - echo "Trying Renovate image candidate: ${candidate}" + echo "Trying Renovate image candidate: ${candidate}" >&2 if docker image inspect "${candidate}" >/dev/null 2>&1; then - echo "Using cached Renovate image: ${candidate}" - echo "${candidate}" + echo "Using cached Renovate image: ${candidate}" >&2 + printf '%s\n' "${candidate}" return 0 fi if retry_cmd 3 15 docker pull "${candidate}"; then - echo "${candidate}" + printf '%s\n' "${candidate}" return 0 fi - echo "⚠ Renovate image candidate failed: ${candidate}" + echo "⚠ Renovate image candidate failed: ${candidate}" >&2 done return 1 -- 2.49.1 From 35389ad15bdea7cc8b7eb8b6e0f81062e160b0ce Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 09:31:04 -0400 Subject: [PATCH 67/80] Expand mirror repair script for Renovate and Playwright images --- docs/GITEA_ACTIONS_TROUBLESHOOTING.md | 6 +- .../gitea-actions/repair_runner_mirror.xsh | 74 +++++++++++++------ 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md index 15bc1c0..4bec6d4 100644 --- a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md +++ b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md @@ -175,7 +175,11 @@ If you want a failover strategy, keep the cached image tagged in the local regis Automation scripts for this workflow live in `scripts/gitea-actions/`: - `scripts/gitea-actions/repair_runner_mirror.xsh` - Repairs the mirror by pulling from GHCR, tagging/pushing to local registry, and verifying tag presence on each runner host. + Repairs mirrors by pulling upstream images, tagging/pushing to local registry, and verifying pullability on each runner host. + Current mirror targets: + - `kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest` + - `kankali.darkhelm.lan:3001/darkhelm.org/renovate:41` + - `kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy` - `scripts/gitea-actions/check_runner_images.xsh` Verifies host-by-host image presence and pullability for `ubuntu:22.04`, upstream act image (`GHCR`), mirrored act image (`MIRROR`), and Renovate image. diff --git a/scripts/gitea-actions/repair_runner_mirror.xsh b/scripts/gitea-actions/repair_runner_mirror.xsh index 51dbbf7..74ca069 100644 --- a/scripts/gitea-actions/repair_runner_mirror.xsh +++ b/scripts/gitea-actions/repair_runner_mirror.xsh @@ -5,36 +5,64 @@ hosts = [ "pi-desktop.darkhelm.lan", ] -# Pull upstream tag, retag/push to local registry mirror, verify mirror tag. +# Pull upstream tags, retag/push to local registry mirrors, and verify pullability. remote_code = """ +mirror_pairs = [ + ("ACT_UBUNTU", "ghcr.io/catthehacker/ubuntu:act-latest", "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest"), + ("RENOVATE", "ghcr.io/renovatebot/renovate:41", "kankali.darkhelm.lan:3001/darkhelm.org/renovate:41"), + ("PLAYWRIGHT", "mcr.microsoft.com/playwright:v1.56.1-jammy", "kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy"), +] + +def classify_failure(prefix, log_path): + mismatch = !(grep -qi "http response to https client" @(log_path)) + print(f"{prefix}:https-mismatch" if mismatch.returncode == 0 else f"{prefix}:failed") + $(docker pull ubuntu:22.04) -$(docker pull ghcr.io/catthehacker/ubuntu:act-latest) -$(docker tag ghcr.io/catthehacker/ubuntu:act-latest kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest) - -push = !(docker push kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-push.log 2>&1) -if push.returncode == 0: - print("MIRROR_PUSH:ok") -else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-push.log) - print("MIRROR_PUSH:https-mismatch" if mismatch.returncode == 0 else "MIRROR_PUSH:failed") - -# Validate that the mirror is actually pullable from this host's Docker configuration. -$(docker image rm kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null || true) -pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-verify.log 2>&1) -if pull.returncode == 0: - print("MIRROR_PULL:ok") -else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-verify.log) - print("MIRROR_PULL:https-mismatch" if mismatch.returncode == 0 else "MIRROR_PULL:failed") if $(docker image inspect ubuntu:22.04): print("UBUNTU22_PRESENT") else: print("UBUNTU22_MISSING") -if $(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest): - print("MIRROR_PRESENT") -else: - print("MIRROR_MISSING") + +for label, source_image, mirror_image in mirror_pairs: + print(f"{label}_SOURCE={source_image}") + print(f"{label}_MIRROR={mirror_image}") + + src_pull_log = f"/tmp/{label.lower()}-source-pull.log" + mirror_push_log = f"/tmp/{label.lower()}-mirror-push.log" + mirror_verify_log = f"/tmp/{label.lower()}-mirror-verify.log" + + src_pull = !(docker pull @(source_image) > @(src_pull_log) 2>&1) + if src_pull.returncode == 0: + print(f"{label}_SOURCE_PULL:ok") + else: + classify_failure(f"{label}_SOURCE_PULL", src_pull_log) + continue + + tag = !(docker tag @(source_image) @(mirror_image)) + if tag.returncode != 0: + print(f"{label}_TAG:failed") + continue + + push = !(docker push @(mirror_image) > @(mirror_push_log) 2>&1) + if push.returncode == 0: + print(f"{label}_MIRROR_PUSH:ok") + else: + classify_failure(f"{label}_MIRROR_PUSH", mirror_push_log) + continue + + # Validate mirror pullability with this host's Docker registry config. + $(docker image rm @(mirror_image) > /dev/null 2> /dev/null || true) + verify = !(docker pull @(mirror_image) > @(mirror_verify_log) 2>&1) + if verify.returncode == 0: + print(f"{label}_MIRROR_PULL:ok") + else: + classify_failure(f"{label}_MIRROR_PULL", mirror_verify_log) + + if $(docker image inspect @(mirror_image)): + print(f"{label}_MIRROR_PRESENT") + else: + print(f"{label}_MIRROR_MISSING") """ for host in hosts: -- 2.49.1 From 32b218a97bcce0781d2eca59198cd2df81c2430c Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 09:33:45 -0400 Subject: [PATCH 68/80] Add xonsh shebang for repair_runner_mirror script --- scripts/gitea-actions/repair_runner_mirror.xsh | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 scripts/gitea-actions/repair_runner_mirror.xsh diff --git a/scripts/gitea-actions/repair_runner_mirror.xsh b/scripts/gitea-actions/repair_runner_mirror.xsh old mode 100644 new mode 100755 index 74ca069..291dc10 --- a/scripts/gitea-actions/repair_runner_mirror.xsh +++ b/scripts/gitea-actions/repair_runner_mirror.xsh @@ -1,3 +1,5 @@ +#!/usr/bin/env xonsh + hosts = [ "kankali.darkhelm.lan", "zhokq.darkhelm.lan", -- 2.49.1 From 6630805d314730dc66890c575422775a343ece8e Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 12:40:46 -0400 Subject: [PATCH 69/80] Fix xonsh syntax in mirror repair remote snippet --- scripts/gitea-actions/repair_runner_mirror.xsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitea-actions/repair_runner_mirror.xsh b/scripts/gitea-actions/repair_runner_mirror.xsh index 291dc10..39c4cad 100755 --- a/scripts/gitea-actions/repair_runner_mirror.xsh +++ b/scripts/gitea-actions/repair_runner_mirror.xsh @@ -54,7 +54,7 @@ for label, source_image, mirror_image in mirror_pairs: continue # Validate mirror pullability with this host's Docker registry config. - $(docker image rm @(mirror_image) > /dev/null 2> /dev/null || true) + rm_image = !(docker image rm @(mirror_image) > /dev/null 2>&1) verify = !(docker pull @(mirror_image) > @(mirror_verify_log) 2>&1) if verify.returncode == 0: print(f"{label}_MIRROR_PULL:ok") -- 2.49.1 From 5232db72897bb4a32016bd501c4a7e1fb71fcbec Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 17:07:58 -0400 Subject: [PATCH 70/80] Publish mirrors only from kankali and improve diagnostics --- .../gitea-actions/repair_runner_mirror.xsh | 68 +++++++++++++------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/scripts/gitea-actions/repair_runner_mirror.xsh b/scripts/gitea-actions/repair_runner_mirror.xsh index 39c4cad..6904ac0 100755 --- a/scripts/gitea-actions/repair_runner_mirror.xsh +++ b/scripts/gitea-actions/repair_runner_mirror.xsh @@ -7,21 +7,42 @@ hosts = [ "pi-desktop.darkhelm.lan", ] -# Pull upstream tags, retag/push to local registry mirrors, and verify pullability. -remote_code = """ +publisher_host = "kankali.darkhelm.lan" + +# Pull upstream tags on the registry host, publish mirror tags there, then verify pullability on every host. +remote_code_template = """ mirror_pairs = [ ("ACT_UBUNTU", "ghcr.io/catthehacker/ubuntu:act-latest", "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest"), ("RENOVATE", "ghcr.io/renovatebot/renovate:41", "kankali.darkhelm.lan:3001/darkhelm.org/renovate:41"), ("PLAYWRIGHT", "mcr.microsoft.com/playwright:v1.56.1-jammy", "kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy"), ] +is_publisher = __IS_PUBLISHER__ + def classify_failure(prefix, log_path): mismatch = !(grep -qi "http response to https client" @(log_path)) - print(f"{prefix}:https-mismatch" if mismatch.returncode == 0 else f"{prefix}:failed") + if mismatch.returncode == 0: + print(f"{prefix}:https-mismatch") + else: + print(f"{prefix}:failed") + tail = !(tail -n 20 @(log_path) 2> /dev/null) + if tail.returncode == 0 and str(tail.out).strip(): + print(f"{prefix}_LOG_START") + print(str(tail.out).rstrip()) + print(f"{prefix}_LOG_END") -$(docker pull ubuntu:22.04) +def image_present(image): + result = !(docker image inspect @(image) > /dev/null 2>&1) + return result.returncode == 0 -if $(docker image inspect ubuntu:22.04): +ubuntu_pull_log = "/tmp/ubuntu22-pull.log" +ubuntu_pull = !(docker pull ubuntu:22.04 > @(ubuntu_pull_log) 2>&1) +if ubuntu_pull.returncode == 0: + print("UBUNTU22_PULL:ok") +else: + classify_failure("UBUNTU22_PULL", ubuntu_pull_log) + +if image_present("ubuntu:22.04"): print("UBUNTU22_PRESENT") else: print("UBUNTU22_MISSING") @@ -34,24 +55,28 @@ for label, source_image, mirror_image in mirror_pairs: mirror_push_log = f"/tmp/{label.lower()}-mirror-push.log" mirror_verify_log = f"/tmp/{label.lower()}-mirror-verify.log" - src_pull = !(docker pull @(source_image) > @(src_pull_log) 2>&1) - if src_pull.returncode == 0: - print(f"{label}_SOURCE_PULL:ok") - else: - classify_failure(f"{label}_SOURCE_PULL", src_pull_log) - continue + if is_publisher: + src_pull = !(docker pull @(source_image) > @(src_pull_log) 2>&1) + if src_pull.returncode == 0: + print(f"{label}_SOURCE_PULL:ok") + else: + classify_failure(f"{label}_SOURCE_PULL", src_pull_log) + continue - tag = !(docker tag @(source_image) @(mirror_image)) - if tag.returncode != 0: - print(f"{label}_TAG:failed") - continue + tag = !(docker tag @(source_image) @(mirror_image) > /dev/null 2>&1) + if tag.returncode != 0: + print(f"{label}_TAG:failed") + continue - push = !(docker push @(mirror_image) > @(mirror_push_log) 2>&1) - if push.returncode == 0: - print(f"{label}_MIRROR_PUSH:ok") + push = !(docker push @(mirror_image) > @(mirror_push_log) 2>&1) + if push.returncode == 0: + print(f"{label}_MIRROR_PUSH:ok") + else: + classify_failure(f"{label}_MIRROR_PUSH", mirror_push_log) + continue else: - classify_failure(f"{label}_MIRROR_PUSH", mirror_push_log) - continue + print(f"{label}_SOURCE_PULL:skipped") + print(f"{label}_MIRROR_PUSH:skipped") # Validate mirror pullability with this host's Docker registry config. rm_image = !(docker image rm @(mirror_image) > /dev/null 2>&1) @@ -61,7 +86,7 @@ for label, source_image, mirror_image in mirror_pairs: else: classify_failure(f"{label}_MIRROR_PULL", mirror_verify_log) - if $(docker image inspect @(mirror_image)): + if image_present(mirror_image): print(f"{label}_MIRROR_PRESENT") else: print(f"{label}_MIRROR_MISSING") @@ -69,4 +94,5 @@ for label, source_image, mirror_image in mirror_pairs: for host in hosts: print(f"\n=== {host} ===") + remote_code = remote_code_template.replace("__IS_PUBLISHER__", "True" if host == publisher_host else "False") ssh @(host) @(remote_code) -- 2.49.1 From 9bea83499999c999572c7ae9bd8aa7ff307c8c64 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 17:37:45 -0400 Subject: [PATCH 71/80] Silence renovate image selector stdout contamination --- .gitea/workflows/renovate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index d0fa22a..bb497b3 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -42,14 +42,14 @@ jobs: if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)) - echo "Command failed (attempt ${attempt}/${attempts}): $*" - echo "Retrying in ${sleep_seconds}s" + echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2 + echo "Retrying in ${sleep_seconds}s" >&2 sleep "${sleep_seconds}" fi attempt=$((attempt + 1)) done - echo "Command failed after ${attempts} attempts: $*" + echo "Command failed after ${attempts} attempts: $*" >&2 return 1 } @@ -70,7 +70,7 @@ jobs: return 0 fi - if retry_cmd 3 15 docker pull "${candidate}"; then + if retry_cmd 3 15 docker pull "${candidate}" >/dev/null; then printf '%s\n' "${candidate}" return 0 fi -- 2.49.1 From efe1a7ee50b1277d352974e4198d1258e47661e3 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 11 Jul 2026 23:10:20 -0400 Subject: [PATCH 72/80] Stabilize Renovate manager scope and remove OSV fetch --- renovate.json | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/renovate.json b/renovate.json index 205c4d0..d89213e 100644 --- a/renovate.json +++ b/renovate.json @@ -11,7 +11,14 @@ "prConcurrentLimit": 5, "branchConcurrentLimit": 10, "prHourlyLimit": 2, - + "enabledManagers": [ + "docker-compose", + "dockerfile", + "github-actions", + "npm", + "pep621", + "regex" + ], "packageRules": [ { "description": "Automerge non-major updates for high-confidence packages", @@ -28,9 +35,24 @@ }, { "description": "Group Python dev tools updates", - "matchManagers": ["uv"], + "matchManagers": ["pep621"], "matchCategories": ["python"], - "matchDepTypes": ["dev-dependencies"], + "matchPackageNames": [ + "ruff", + "pyright", + "pydoclint", + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "pre-commit", + "pyyaml", + "yamllint", + "toml-sort", + "language-formatters-pre-commit-hooks", + "xdoctest", + "poethepoet" + ], "groupName": "Python dev tools", "schedule": ["before 9am on monday"] }, @@ -76,7 +98,6 @@ "reviewersFromCodeOwners": true } ], - "customManagers": [ { "description": "Update Python version in Dockerfiles", @@ -97,19 +118,15 @@ "versioningTemplate": "node" } ], - - "osvVulnerabilityAlerts": true, + "osvVulnerabilityAlerts": false, "vulnerabilityAlerts": { "enabled": true, "schedule": ["at any time"] }, - "labels": ["dependencies"], "commitMessagePrefix": "chore:", "commitMessageTopic": "{{depName}}", "commitMessageExtra": "to {{newVersion}}", "commitMessageSuffix": "", - - "prTitle": "{{commitMessagePrefix}} {{commitMessageAction}} {{commitMessageTopic}} {{commitMessageExtra}}", "prBodyTemplate": "This PR contains the following updates:\n\n| Package | Change | Age | Adoption | Passing | Confidence |\n|---|---|---|---|---|---|\n{{#each upgrades}}\n|{{depName}}|{{#if displayFrom}}`{{{displayFrom}}}` -> `{{{displayTo}}}`{{else}}`{{{newVersion}}}`{{/if}}|[![age](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/age-slim)](https://docs.renovatebot.com/merge-confidence/)|[![adoption](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/adoption-slim)](https://docs.renovatebot.com/merge-confidence/)|[![passing](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/compatibility-slim/{{currentVersion}})](https://docs.renovatebot.com/merge-confidence/)|[![confidence](https://badges.renovateapi.com/packages/{{datasource}}/{{depName}}/{{newVersion}}/confidence-slim/{{currentVersion}})](https://docs.renovatebot.com/merge-confidence/)|\n{{/each}}\n\n---\n\n### Release Notes\n\n{{#each upgrades}}\n{{#if hasReleaseNotes}}\n
\n{{depName}}\n\n{{#each releases}}\n#### {{title}}\n\n{{#if body}}\n{{body}}\n{{/if}}\n{{/each}}\n\n
\n{{/if}}\n{{/each}}\n\n### Configuration\n\n📅 **Schedule**: {{schedule}}\n\n🚦 **Automerge**: {{#if isAutomerge}}Enabled{{else}}Disabled{{/if}}\n\n♻ **Rebasing**: {{#if isRebasing}}Rebasing{{else}}Not rebasing{{/if}}\n\n👥 **Reviewers**: {{#if reviewers}}{{reviewers}}{{else}}None{{/if}}\n\n---\n\n*This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate).*" } -- 2.49.1 From 7e564678255fb916e6b3b6219c2910202540ffbb Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 12 Jul 2026 07:14:32 -0400 Subject: [PATCH 73/80] Reduce Renovate memory pressure and skip OOM retry --- .gitea/workflows/renovate.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index bb497b3..c328a69 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -118,8 +118,13 @@ jobs: RENOVATE_REQUIRE_CONFIG: required RENOVATE_PR_CONCURRENT_LIMIT: "3" RENOVATE_BRANCH_CONCURRENT_LIMIT: "5" + RENOVATE_NODE_ARGS: --max-old-space-size=768 + RENOVATE_OSV_VULNERABILITY_ALERTS: "false" + RENOVATE_VULNERABILITY_ALERTS: "false" LOG_LEVEL: debug run: | + set -euo pipefail + echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" @@ -349,6 +354,9 @@ jobs: -e RENOVATE_REQUIRE_CONFIG \ -e RENOVATE_PR_CONCURRENT_LIMIT \ -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ + -e RENOVATE_NODE_ARGS \ + -e RENOVATE_OSV_VULNERABILITY_ALERTS \ + -e RENOVATE_VULNERABILITY_ALERTS \ --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ -lc 'set -e @@ -359,12 +367,17 @@ jobs: git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/" git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}" echo "Configured git dogar->kankali rewrite rules" - renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" DarkHelm.org/plex-playlist' + renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" --vulnerability-alerts="${RENOVATE_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' } if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then : else + primary_status=$? + if [ "${primary_status}" -eq 137 ] || grep -qE '(^|[[:space:]])Killed($|[[:space:]])|exitcode '\''137'\''' /tmp/renovate.log; then + echo "❌ Renovate terminated by OOM/kill signal on primary endpoint; skipping endpoint retry" + exit "${primary_status}" + fi echo "⚠ Renovate run failed with primary endpoint. Retrying with API endpoint form..." run_renovate_with_endpoint "${API_ENDPOINT}" 2>&1 | tee /tmp/renovate.log fi -- 2.49.1 From a0903cf7973477b3a7e636fc99161ab8771d92a0 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 12 Jul 2026 07:38:31 -0400 Subject: [PATCH 74/80] Remove unsupported Renovate vulnerability-alerts flag --- .gitea/workflows/renovate.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index c328a69..da978ea 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -120,7 +120,6 @@ jobs: RENOVATE_BRANCH_CONCURRENT_LIMIT: "5" RENOVATE_NODE_ARGS: --max-old-space-size=768 RENOVATE_OSV_VULNERABILITY_ALERTS: "false" - RENOVATE_VULNERABILITY_ALERTS: "false" LOG_LEVEL: debug run: | set -euo pipefail @@ -356,7 +355,6 @@ jobs: -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ -e RENOVATE_NODE_ARGS \ -e RENOVATE_OSV_VULNERABILITY_ALERTS \ - -e RENOVATE_VULNERABILITY_ALERTS \ --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ -lc 'set -e @@ -367,7 +365,7 @@ jobs: git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/" git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}" echo "Configured git dogar->kankali rewrite rules" - renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" --vulnerability-alerts="${RENOVATE_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' + renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' } if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then -- 2.49.1 From ff184023209d66565d85c013809931ea8172fe41 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 12 Jul 2026 08:48:22 -0400 Subject: [PATCH 75/80] Force Renovate to use workspace config --- .gitea/workflows/renovate.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index da978ea..1631e9c 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -127,6 +127,12 @@ jobs: echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" + WORKSPACE_CONFIG_PATH="${GITHUB_WORKSPACE:-$PWD}/renovate.json" + + if [ ! -f "${WORKSPACE_CONFIG_PATH}" ]; then + echo "❌ Expected Renovate config not found at ${WORKSPACE_CONFIG_PATH}" + exit 1 + fi CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then @@ -355,6 +361,7 @@ jobs: -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ -e RENOVATE_NODE_ARGS \ -e RENOVATE_OSV_VULNERABILITY_ALERTS \ + -v "${WORKSPACE_CONFIG_PATH}:/tmp/renovate-config.json:ro" \ --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ -lc 'set -e @@ -365,7 +372,7 @@ jobs: git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/" git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}" echo "Configured git dogar->kankali rewrite rules" - renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' + renovate --config-file /tmp/renovate-config.json --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' } if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then -- 2.49.1 From b807dcb115e52e3a392443b0b9d499e72eb063f6 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 13 Jul 2026 00:42:59 -0400 Subject: [PATCH 76/80] Load Renovate config from git HEAD --- .gitea/workflows/renovate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 1631e9c..2df7e93 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -127,10 +127,10 @@ jobs: echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" - WORKSPACE_CONFIG_PATH="${GITHUB_WORKSPACE:-$PWD}/renovate.json" + RENOVATE_CONFIG_PATH="/tmp/renovate-config.json" - if [ ! -f "${WORKSPACE_CONFIG_PATH}" ]; then - echo "❌ Expected Renovate config not found at ${WORKSPACE_CONFIG_PATH}" + if ! git show HEAD:renovate.json > "${RENOVATE_CONFIG_PATH}"; then + echo "❌ Expected Renovate config not found in git HEAD: renovate.json" exit 1 fi @@ -361,7 +361,7 @@ jobs: -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ -e RENOVATE_NODE_ARGS \ -e RENOVATE_OSV_VULNERABILITY_ALERTS \ - -v "${WORKSPACE_CONFIG_PATH}:/tmp/renovate-config.json:ro" \ + -v "${RENOVATE_CONFIG_PATH}:/tmp/renovate-config.json:ro" \ --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ -lc 'set -e -- 2.49.1 From 64903d0343b8780c633700eff2e0b80a8f1c58e2 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 13 Jul 2026 09:44:53 -0400 Subject: [PATCH 77/80] Fix Renovate config fetch YAML syntax --- .gitea/workflows/renovate.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 2df7e93..aafd8d5 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -105,6 +105,7 @@ jobs: RENOVATE_TOKEN_SECRET: ${{ secrets.RENOVATE_TOKEN }} ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + RENOVATE_CONFIG_REF: ${{ github.ref_name }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_PLATFORM: gitea RENOVATE_ENDPOINT_INTERNAL: http://kankali.darkhelm.lan:3001/api/v1 @@ -128,11 +129,7 @@ jobs: TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" RENOVATE_CONFIG_PATH="/tmp/renovate-config.json" - - if ! git show HEAD:renovate.json > "${RENOVATE_CONFIG_PATH}"; then - echo "❌ Expected Renovate config not found in git HEAD: renovate.json" - exit 1 - fi + RENOVATE_CONFIG_PAYLOAD="/tmp/renovate-config-payload.json" CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then @@ -219,6 +216,25 @@ jobs: echo "Renovate git base URL override: ${RENOVATE_GIT_URL}" echo "Preflight API endpoint: ${API_ENDPOINT}" + CONFIG_FETCH_TOKEN="${RENOVATE_TOKEN_SECRET:-${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}}" + if [ -z "${CONFIG_FETCH_TOKEN}" ]; then + echo "❌ No token available to fetch renovate.json from the repo contents API" + exit 1 + fi + + if ! curl -sS ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${CONFIG_FETCH_TOKEN}" \ + "${API_ENDPOINT}/repos/${TARGET_REPO}/contents/renovate.json?ref=${RENOVATE_CONFIG_REF}" \ + -o "${RENOVATE_CONFIG_PAYLOAD}"; then + echo "❌ Expected Renovate config could not be fetched from the repo contents API at ref ${RENOVATE_CONFIG_REF}" + exit 1 + fi + + if ! python3 -c 'import base64, json, sys; output_path = sys.argv[1]; payload_path = sys.argv[2]; payload = json.load(open(payload_path, encoding="utf-8")); content = payload.get("content"); raise SystemExit(1) if not content else None; open(output_path, "wb").write(base64.b64decode(content.replace("\n", "")))' "${RENOVATE_CONFIG_PATH}" "${RENOVATE_CONFIG_PAYLOAD}"; then + echo "❌ Expected Renovate config could not be decoded from the repo contents API response at ref ${RENOVATE_CONFIG_REF}" + exit 1 + fi + PLATFORM_VERSION="" if [ "${VERSION_STATUS}" = "200" ]; then PLATFORM_VERSION=$(sed -n 's/.*"version":"\([^"]*\)".*/\1/p' /tmp/renovate-version-check.json | head -n 1) -- 2.49.1 From 0e7c4c8e91a79c77c49da64c69d83fd89a22d27a Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 13 Jul 2026 09:52:20 -0400 Subject: [PATCH 78/80] Fix Renovate config decoder YAML --- .gitea/workflows/renovate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index aafd8d5..dd096bc 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -230,7 +230,7 @@ jobs: exit 1 fi - if ! python3 -c 'import base64, json, sys; output_path = sys.argv[1]; payload_path = sys.argv[2]; payload = json.load(open(payload_path, encoding="utf-8")); content = payload.get("content"); raise SystemExit(1) if not content else None; open(output_path, "wb").write(base64.b64decode(content.replace("\n", "")))' "${RENOVATE_CONFIG_PATH}" "${RENOVATE_CONFIG_PAYLOAD}"; then + if ! python3 -c 'import base64, json, sys; output_path = sys.argv[1]; payload_path = sys.argv[2]; payload = json.load(open(payload_path, encoding="utf-8")); content = payload.get("content"); if not content: raise SystemExit(1); open(output_path, "wb").write(base64.b64decode(content.replace("\n", "")))' "${RENOVATE_CONFIG_PATH}" "${RENOVATE_CONFIG_PAYLOAD}"; then echo "❌ Expected Renovate config could not be decoded from the repo contents API response at ref ${RENOVATE_CONFIG_REF}" exit 1 fi -- 2.49.1 From ed6501a450c377da5f66bea18378cd39381fdd8b Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 13 Jul 2026 10:06:06 -0400 Subject: [PATCH 79/80] Fix Renovate config decoder syntax --- .gitea/workflows/renovate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index dd096bc..e4be061 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -230,7 +230,7 @@ jobs: exit 1 fi - if ! python3 -c 'import base64, json, sys; output_path = sys.argv[1]; payload_path = sys.argv[2]; payload = json.load(open(payload_path, encoding="utf-8")); content = payload.get("content"); if not content: raise SystemExit(1); open(output_path, "wb").write(base64.b64decode(content.replace("\n", "")))' "${RENOVATE_CONFIG_PATH}" "${RENOVATE_CONFIG_PAYLOAD}"; then + if ! python3 -c 'import base64, json, sys; output_path = sys.argv[1]; payload_path = sys.argv[2]; payload = json.load(open(payload_path, encoding="utf-8")); content = payload.get("content"); sys.exit(1) if not content else None; open(output_path, "wb").write(base64.b64decode(content.replace("\n", "")))' "${RENOVATE_CONFIG_PATH}" "${RENOVATE_CONFIG_PAYLOAD}"; then echo "❌ Expected Renovate config could not be decoded from the repo contents API response at ref ${RENOVATE_CONFIG_REF}" exit 1 fi -- 2.49.1 From a1e977379913b0d83be21c24324958db8a6285b1 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Mon, 13 Jul 2026 10:33:27 -0400 Subject: [PATCH 80/80] Fix Renovate workflow cleanup --- .gitea/workflows/renovate.yml | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index e4be061..da978ea 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -105,7 +105,6 @@ jobs: RENOVATE_TOKEN_SECRET: ${{ secrets.RENOVATE_TOKEN }} ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - RENOVATE_CONFIG_REF: ${{ github.ref_name }} RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_PLATFORM: gitea RENOVATE_ENDPOINT_INTERNAL: http://kankali.darkhelm.lan:3001/api/v1 @@ -128,8 +127,6 @@ jobs: echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" TARGET_ORG="${TARGET_REPO%%/*}" - RENOVATE_CONFIG_PATH="/tmp/renovate-config.json" - RENOVATE_CONFIG_PAYLOAD="/tmp/renovate-config-payload.json" CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then @@ -216,25 +213,6 @@ jobs: echo "Renovate git base URL override: ${RENOVATE_GIT_URL}" echo "Preflight API endpoint: ${API_ENDPOINT}" - CONFIG_FETCH_TOKEN="${RENOVATE_TOKEN_SECRET:-${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}}" - if [ -z "${CONFIG_FETCH_TOKEN}" ]; then - echo "❌ No token available to fetch renovate.json from the repo contents API" - exit 1 - fi - - if ! curl -sS ${CURL_INSECURE_FLAG} \ - -H "Authorization: token ${CONFIG_FETCH_TOKEN}" \ - "${API_ENDPOINT}/repos/${TARGET_REPO}/contents/renovate.json?ref=${RENOVATE_CONFIG_REF}" \ - -o "${RENOVATE_CONFIG_PAYLOAD}"; then - echo "❌ Expected Renovate config could not be fetched from the repo contents API at ref ${RENOVATE_CONFIG_REF}" - exit 1 - fi - - if ! python3 -c 'import base64, json, sys; output_path = sys.argv[1]; payload_path = sys.argv[2]; payload = json.load(open(payload_path, encoding="utf-8")); content = payload.get("content"); sys.exit(1) if not content else None; open(output_path, "wb").write(base64.b64decode(content.replace("\n", "")))' "${RENOVATE_CONFIG_PATH}" "${RENOVATE_CONFIG_PAYLOAD}"; then - echo "❌ Expected Renovate config could not be decoded from the repo contents API response at ref ${RENOVATE_CONFIG_REF}" - exit 1 - fi - PLATFORM_VERSION="" if [ "${VERSION_STATUS}" = "200" ]; then PLATFORM_VERSION=$(sed -n 's/.*"version":"\([^"]*\)".*/\1/p' /tmp/renovate-version-check.json | head -n 1) @@ -377,7 +355,6 @@ jobs: -e RENOVATE_BRANCH_CONCURRENT_LIMIT \ -e RENOVATE_NODE_ARGS \ -e RENOVATE_OSV_VULNERABILITY_ALERTS \ - -v "${RENOVATE_CONFIG_PATH}:/tmp/renovate-config.json:ro" \ --entrypoint /bin/sh \ "${RENOVATE_IMAGE}" \ -lc 'set -e @@ -388,7 +365,7 @@ jobs: git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/" git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}" echo "Configured git dogar->kankali rewrite rules" - renovate --config-file /tmp/renovate-config.json --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' + renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist' } if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then -- 2.49.1