From c996d92406216b05bb38a553869027d09138070f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 2 Jul 2026 08:47:42 -0400 Subject: [PATCH 01/23] feat(ci): replace integration lane with runtime black-box tests --- .gitea/workflows/cicd-tests.yaml | 173 ++++++++++++++++++++++-- .gitea/workflows/docker-build-main.yaml | 44 ++++++ docs/CICD_SUCCESS_SUMMARY.md | 4 + docs/DEPLOYABLE_RUNTIME_CONTRACT.md | 6 + 4 files changed, 212 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index de807a8..315ec50 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -6,6 +6,12 @@ on: head_sha: description: Commit SHA to process required: false + deployable_backend_tag_ref: + description: Deployable backend image tag reference + required: false + deployable_backend_digest_ref: + description: Deployable backend image digest reference + required: false source_workflow: description: Upstream workflow name required: false @@ -30,6 +36,8 @@ jobs: timeout-minutes: 8 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner run: | @@ -48,6 +56,8 @@ jobs: EVENT_NAME: ${{ github.event_name }} SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} REF: ${{ github.ref }} REF_NAME: ${{ github.ref_name }} @@ -61,6 +71,8 @@ jobs: echo "source_workflow=${SOURCE_WORKFLOW}" echo "head_sha_input=${HEAD_SHA_INPUT}" echo "head_sha=${RESOLVED_HEAD_SHA}" + echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" + echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" echo "ref=${REF}" echo "ref_name=${REF_NAME}" echo "head_ref=${HEAD_REF}" @@ -70,10 +82,14 @@ jobs: id: meta env: HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} run: | RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT" - &failure_diagnostics_step name: Failure diagnostics @@ -269,7 +285,7 @@ jobs: - *failure_diagnostics_step integration-tests: - name: Integration Tests + name: Runtime Black-Box Integration Tests # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 @@ -288,28 +304,155 @@ jobs: echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run integration tests + - name: Run runtime black-box integration checks env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }} run: | + set -euo pipefail set -o pipefail - LOG_FILE="$(mktemp)" - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - if [ -d 'tests/integration' ]; then - uv run pytest tests/integration/ -v --tb=short - else - echo 'No integration tests found' + LOG_FILE="$(mktemp)" + NETWORK_NAME="plex-blackbox-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + DB_CONTAINER="plex-blackbox-db-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + BACKEND_CONTAINER="plex-blackbox-backend-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" + + dump_failure_context() { + echo "=== Runtime Integration Failure Context ===" + docker ps -a || true + echo "--- Backend logs (tail 200) ---" + docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Database logs (tail 200) ---" + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Backend inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true + echo "--- Database inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true + } + + cleanup() { + docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true + docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true + docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + { + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable backend image references from dispatch inputs" + exit 1 fi - " 2>&1 | tee "${LOG_FILE}" + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then + echo "❌ Tag and digest mismatch" + echo "expected=${EXPECTED_DIGEST_REF}" + echo "actual=${ACTUAL_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" + echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" + + docker network create "${NETWORK_NAME}" + + docker run -d \ + --name "${DB_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -e POSTGRES_DB=plex_playlist \ + -e POSTGRES_USER=plex_user \ + -e POSTGRES_PASSWORD="${DB_PASSWORD}" \ + postgres:16-alpine + + db_ready=false + for i in $(seq 1 30); do + if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + docker run -d \ + --name "${BACKEND_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -p 18000:8000 \ + -e DATABASE_URL="${DB_URL}" \ + -e ENVIRONMENT=production \ + "${EXPECTED_DIGEST_REF}" + + backend_ready=false + for i in $(seq 1 40); do + health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(curl -sS -o /tmp/blackbox-root.json -w "%{http_code}" "http://127.0.0.1:18000/" || true)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + compatibility_code="$(curl -sS -o /tmp/blackbox-compatibility.json -w "%{http_code}" "http://127.0.0.1:18000/compatibility" || true)" + if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" + if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then + echo "❌ Health endpoint validation failed" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + echo "✅ Runtime black-box integration checks passed" + } 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Integration tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of integration test output ---" + echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of runtime black-box integration output ---" tail -n 200 "${LOG_FILE}" || true exit "${TEST_STATUS}" fi diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml index 5ff3303..bdf21c2 100644 --- a/.gitea/workflows/docker-build-main.yaml +++ b/.gitea/workflows/docker-build-main.yaml @@ -138,6 +138,8 @@ jobs: timeout-minutes: 60 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner @@ -251,6 +253,44 @@ jobs: --image deployable-frontend:"${HEAD_SHA}" \ --profile frontend + - name: Push deployable backend runtime image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker push "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve deployable backend digest reference" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" + - name: Build and push complete CICD image env: PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} @@ -347,6 +387,8 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} HEAD_SHA: ${{ needs.build.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }} REPO_FULL: ${{ github.repository }} HEAD_REF: ${{ github.head_ref }} REF_NAME: ${{ github.ref_name }} @@ -426,6 +468,8 @@ jobs: --head-sha "${HEAD_SHA}" --source-workflow "CICD Main Build" --trace-id "${TRACE_ID}" + --input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + --input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" ) for API_BASE in "${CANDIDATE_API_BASES[@]}"; do diff --git a/docs/CICD_SUCCESS_SUMMARY.md b/docs/CICD_SUCCESS_SUMMARY.md index de4902b..a9decc3 100644 --- a/docs/CICD_SUCCESS_SUMMARY.md +++ b/docs/CICD_SUCCESS_SUMMARY.md @@ -33,6 +33,8 @@ - **E2E Tests**: Simplified Docker approach matching other successful test patterns - **Playwright**: Chromium-only CI strategy (95%+ browser market coverage) - **Registry Operations**: Consistent approach across all test phases +- **Backend Runtime Integration**: Black-box checks run against started deployable + backend containers with commit-tag and digest pinning ## 🛠️ **Critical Issues Resolved** @@ -82,6 +84,8 @@ Frontend Environment (Yarn PnP) ↓ (state regeneration) Test Execution (all phases) ↓ (consistent Docker approach) +Runtime Black-Box Integration + ↓ (deployable backend tag+digest verification) E2E Testing (Playwright) ↓ (Chromium + network resilience) ✅ SUCCESS diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 64ddf00..1d9e999 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -157,6 +157,12 @@ Current enforcement implemented in CI: - Workflow: `.gitea/workflows/docker-build-main.yaml` - Checks include binary presence and profile-specific package metadata probes to detect CI/development tooling leakage. +- Runtime black-box integration checks against deployed backend container: + - Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job) + - Inputs: deployable backend commit tag reference and immutable digest + reference from main build dispatch. + - Assertions: digest/tag consistency and live endpoint behavior for `/`, + `/compatibility`, and `/health` on started runtime containers. Future follow-up automation under epic #66 may expand this with: -- 2.49.1 From 1295ffb330827f04748b1995bb89d7ec34859bac Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 2 Jul 2026 09:47:33 -0400 Subject: [PATCH 02/23] fix(ci): lock backend deps in cicd build and harden renovate auth --- .gitea/workflows/renovate.yml | 82 +++++++++++++++++++---------------- Dockerfile.cicd | 9 +++- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 0bc2461..010a872 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -242,31 +242,7 @@ jobs: LOG_LEVEL: info run: | echo "=== Running Renovate Bot ===" - - select_token() { - if [ -n "${RENOVATE_TOKEN_SECRET:-}" ]; then - echo "${RENOVATE_TOKEN_SECRET}" - return 0 - fi - if [ -n "${ACTIONS_TRIGGER_TOKEN:-}" ]; then - echo "${ACTIONS_TRIGGER_TOKEN}" - return 0 - fi - if [ -n "${PACKAGE_ACCESS_TOKEN:-}" ]; then - echo "${PACKAGE_ACCESS_TOKEN}" - return 0 - fi - return 1 - } - - if ! SELECTED_TOKEN="$(select_token)"; then - echo "❌ No token available for Renovate authentication" - echo "Configure one of: RENOVATE_TOKEN, ACTIONS_TRIGGER_TOKEN, PACKAGE_ACCESS_TOKEN" - exit 1 - fi - - export RENOVATE_TOKEN="${SELECTED_TOKEN}" - unset SELECTED_TOKEN + TARGET_REPO="DarkHelm.org/plex-playlist" CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then @@ -275,23 +251,55 @@ jobs: export NODE_TLS_REJECT_UNAUTHORIZED=0 fi - # Validate token before starting Renovate to fail with actionable diagnostics. - AUTH_CHECK_STATUS=$(curl -sS -o /tmp/renovate-auth-check.json -w "%{http_code}" \ - ${CURL_INSECURE_FLAG} \ - -H "Authorization: token ${RENOVATE_TOKEN}" \ - "${RENOVATE_ENDPOINT}/user" || true) + select_token_with_repo_access() { + for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do + candidate_value="${!candidate_name:-}" + if [ -z "${candidate_value}" ]; then + continue + fi - if [ "${AUTH_CHECK_STATUS}" != "200" ]; then - echo "❌ Renovate token authentication preflight failed (HTTP ${AUTH_CHECK_STATUS})" - echo "Expected a personal access token with repo/issue read-write permissions." - if [ -s /tmp/renovate-auth-check.json ]; then - echo "Response body:" - cat /tmp/renovate-auth-check.json || true + USER_STATUS=$(curl -sS -o /tmp/renovate-auth-check-user.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${RENOVATE_ENDPOINT}/user" || true) + + REPO_STATUS=$(curl -sS -o /tmp/renovate-auth-check-repo.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${RENOVATE_ENDPOINT}/repos/${TARGET_REPO}" || true) + + if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ]; then + echo "${candidate_name}:${candidate_value}" + return 0 + fi + + echo "⚠ Token candidate ${candidate_name} rejected (user=${USER_STATUS}, repo=${REPO_STATUS})" + done + + return 1 + } + + if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then + echo "❌ No token available for Renovate authentication with repository access" + echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." + echo "Token preflight checks attempted: ${RENOVATE_ENDPOINT}/user and ${RENOVATE_ENDPOINT}/repos/${TARGET_REPO}" + if [ -s /tmp/renovate-auth-check-user.json ]; then + echo "Last user endpoint response body:" + cat /tmp/renovate-auth-check-user.json || true + fi + if [ -s /tmp/renovate-auth-check-repo.json ]; then + echo "Last repo endpoint response body:" + cat /tmp/renovate-auth-check-repo.json || true fi exit 1 fi - echo "✓ Renovate auth preflight passed" + SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" + SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" + export RENOVATE_TOKEN="${SELECTED_TOKEN}" + unset SELECTED_TOKEN RESULT_TOKEN + + echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then diff --git a/Dockerfile.cicd b/Dockerfile.cicd index babe34b..8307b83 100644 --- a/Dockerfile.cicd +++ b/Dockerfile.cicd @@ -55,6 +55,7 @@ RUN --mount=type=secret,id=ssh_private_key \ # Extract only dependency files for caching optimization mkdir -p /workspace/backend /workspace/frontend && \ cp /tmp/repo/backend/pyproject.toml /workspace/backend/ 2>/dev/null || echo "No backend pyproject.toml" && \ + cp /tmp/repo/backend/uv.lock /workspace/backend/ 2>/dev/null || echo "No backend uv.lock" && \ cp /tmp/repo/frontend/package.json /workspace/frontend/ 2>/dev/null || echo "No frontend package.json" && \ cp /tmp/repo/frontend/yarn.lock /workspace/frontend/ 2>/dev/null || echo "No frontend yarn.lock" && \ cp /tmp/repo/frontend/.yarnrc.yml /workspace/frontend/ 2>/dev/null || echo "No frontend .yarnrc.yml" && \ @@ -90,7 +91,13 @@ RUN echo "=== Installing Backend Dependencies (Phase 1: Optimized Caching) ===" echo "# Temporary README for dependency caching phase" > ../README.md && \ echo "# Minimal __init__.py for build" > src/backend/__init__.py && \ # Install all dependencies including dev dependencies - uv sync --dev && \ + if [ -f "uv.lock" ]; then \ + echo "Using locked backend dependencies (uv.lock)" && \ + uv sync --dev --frozen; \ + else \ + echo "uv.lock not found, resolving dependencies from pyproject.toml" && \ + uv sync --dev; \ + fi && \ echo "✓ Backend dependencies installed and cached"; \ else \ echo "No pyproject.toml found, skipping dependency installation"; \ -- 2.49.1 From b1607ff6c7b768f22c57a0a2aba0b24c7c17278a Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 2 Jul 2026 10:28:50 -0400 Subject: [PATCH 03/23] fix(renovate): normalize gitea endpoint and align auth preflight --- .gitea/workflows/renovate.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 010a872..b99ffa1 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -215,7 +215,7 @@ jobs: cat > renovate-config.js << 'EOF' module.exports = { platform: 'gitea', - endpoint: 'https://dogar.darkhelm.org/api/v1', + endpoint: 'https://dogar.darkhelm.org', gitAuthor: 'Renovate Bot ', repositories: ['DarkHelm.org/plex-playlist'], onboarding: false, @@ -237,13 +237,25 @@ jobs: RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_CONFIG_FILE: renovate-config.js RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://dogar.darkhelm.org/api/v1 + RENOVATE_ENDPOINT: https://dogar.darkhelm.org RENOVATE_ALLOW_INSECURE_TLS: "true" LOG_LEVEL: info run: | echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" + RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" + if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then + API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" + else + API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" + fi + + export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + echo "Renovate endpoint: ${RENOVATE_ENDPOINT}" + echo "Preflight API endpoint: ${API_ENDPOINT}" + CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then echo "⚠ Renovate insecure TLS mode enabled for self-signed certificate endpoint" @@ -261,12 +273,12 @@ jobs: USER_STATUS=$(curl -sS -o /tmp/renovate-auth-check-user.json -w "%{http_code}" \ ${CURL_INSECURE_FLAG} \ -H "Authorization: token ${candidate_value}" \ - "${RENOVATE_ENDPOINT}/user" || true) + "${API_ENDPOINT}/user" || true) REPO_STATUS=$(curl -sS -o /tmp/renovate-auth-check-repo.json -w "%{http_code}" \ ${CURL_INSECURE_FLAG} \ -H "Authorization: token ${candidate_value}" \ - "${RENOVATE_ENDPOINT}/repos/${TARGET_REPO}" || true) + "${API_ENDPOINT}/repos/${TARGET_REPO}" || true) if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ]; then echo "${candidate_name}:${candidate_value}" @@ -282,7 +294,7 @@ jobs: if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then echo "❌ No token available for Renovate authentication with repository access" echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." - echo "Token preflight checks attempted: ${RENOVATE_ENDPOINT}/user and ${RENOVATE_ENDPOINT}/repos/${TARGET_REPO}" + echo "Token preflight checks attempted: ${API_ENDPOINT}/user and ${API_ENDPOINT}/repos/${TARGET_REPO}" if [ -s /tmp/renovate-auth-check-user.json ]; then echo "Last user endpoint response body:" cat /tmp/renovate-auth-check-user.json || true -- 2.49.1 From 3a47f3f154d657f19ca33ad103f9d6a94d062308 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 2 Jul 2026 11:09:57 -0400 Subject: [PATCH 04/23] fix(ci): harden runtime integration env and renovate auth checks --- .gitea/workflows/cicd-tests.yaml | 8 +++++--- .gitea/workflows/renovate.yml | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index 315ec50..7e84141 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -312,10 +312,12 @@ jobs: set -euo pipefail set -o pipefail + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" LOG_FILE="$(mktemp)" - NETWORK_NAME="plex-blackbox-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - DB_CONTAINER="plex-blackbox-db-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - BACKEND_CONTAINER="plex-blackbox-backend-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}" + DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}" + BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}" DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index b99ffa1..0202dd4 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -243,6 +243,7 @@ jobs: run: | echo "=== Running Renovate Bot ===" TARGET_REPO="DarkHelm.org/plex-playlist" + TARGET_ORG="${TARGET_REPO%%/*}" RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then @@ -280,12 +281,17 @@ jobs: -H "Authorization: token ${candidate_value}" \ "${API_ENDPOINT}/repos/${TARGET_REPO}" || true) - if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ]; then + ORG_STATUS=$(curl -sS -o /tmp/renovate-auth-check-org.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/orgs/${TARGET_ORG}" || true) + + if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then echo "${candidate_name}:${candidate_value}" return 0 fi - echo "⚠ Token candidate ${candidate_name} rejected (user=${USER_STATUS}, repo=${REPO_STATUS})" + echo "⚠ Token candidate ${candidate_name} rejected (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})" done return 1 @@ -294,7 +300,7 @@ jobs: if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then echo "❌ No token available for Renovate authentication with repository access" echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." - echo "Token preflight checks attempted: ${API_ENDPOINT}/user and ${API_ENDPOINT}/repos/${TARGET_REPO}" + echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}" if [ -s /tmp/renovate-auth-check-user.json ]; then echo "Last user endpoint response body:" cat /tmp/renovate-auth-check-user.json || true @@ -303,6 +309,10 @@ jobs: echo "Last repo endpoint response body:" cat /tmp/renovate-auth-check-repo.json || true fi + if [ -s /tmp/renovate-auth-check-org.json ]; then + echo "Last org endpoint response body:" + cat /tmp/renovate-auth-check-org.json || true + fi exit 1 fi @@ -315,7 +325,10 @@ jobs: # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then + export RENOVATE_DRY_RUN="full" echo "🔍 Running in DRY-RUN mode (no changes will be made)" + else + unset RENOVATE_DRY_RUN fi renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist -- 2.49.1 From 6da60e6beb96ec18c0969e2ddf6cb9bbde8dad95 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Fri, 3 Jul 2026 21:58:36 -0400 Subject: [PATCH 05/23] fix(ci): correct backend ASGI path for runtime black-box lane --- .gitea/workflows/cicd-tests.yaml | 7 +++++++ Dockerfile.backend | 2 +- compose.dev.yml | 2 +- docs/DEPLOYABLE_RUNTIME_CONTRACT.md | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index 7e84141..0fe7b64 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -410,6 +410,13 @@ jobs: backend_ready=false for i in $(seq 1 40); do + backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)" + if [ "${backend_running}" != "true" ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" if [ "${health_code}" = "200" ]; then backend_ready=true diff --git a/Dockerfile.backend b/Dockerfile.backend index 1bec094..51a5a69 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -34,4 +34,4 @@ COPY backend/ . EXPOSE 8000 # Default command - can be overridden in compose for development -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/compose.dev.yml b/compose.dev.yml index 8af76a7..2c99cb3 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -22,7 +22,7 @@ services: - "8001:8000" command: - "uvicorn" - - "main:app" + - "backend.main:app" - "--host" - "0.0.0.0" - "--port" diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 1d9e999..e5123ac 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -31,7 +31,7 @@ Excluded: - Container build source: `Dockerfile.backend`. - Runtime base image: `python:3.14-slim`. -- Runtime process: `uvicorn main:app --host 0.0.0.0 --port 8000`. +- Runtime process: `uvicorn backend.main:app --host 0.0.0.0 --port 8000`. - Exposed runtime port: `8000`. ### Required Runtime Dependencies @@ -68,7 +68,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth. ### Backend Runtime Checklist - [ ] Runtime image built from `Dockerfile.backend`. -- [ ] Runtime process is uvicorn serving `main:app` on `0.0.0.0:8000`. +- [ ] Runtime process is uvicorn serving `backend.main:app` on `0.0.0.0:8000`. - [ ] `DATABASE_URL` is set in deployment runtime. - [ ] `GET /health` behavior matches contract. - [ ] Startup fails on runtime policy mismatch. -- 2.49.1 From 98058475cbf16be15910604b8f21c05582905baf Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 4 Jul 2026 09:15:23 -0400 Subject: [PATCH 06/23] fix(backend): set uvicorn app-dir for src layout runtime --- Dockerfile.backend | 2 +- compose.dev.yml | 2 ++ docs/DEPLOYABLE_RUNTIME_CONTRACT.md | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile.backend b/Dockerfile.backend index 51a5a69..fdc12fa 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -34,4 +34,4 @@ COPY backend/ . EXPOSE 8000 # Default command - can be overridden in compose for development -CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"] diff --git a/compose.dev.yml b/compose.dev.yml index 2c99cb3..2ffb871 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -23,6 +23,8 @@ services: command: - "uvicorn" - "backend.main:app" + - "--app-dir" + - "/app/src" - "--host" - "0.0.0.0" - "--port" diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index e5123ac..0c33cd2 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -31,7 +31,7 @@ Excluded: - Container build source: `Dockerfile.backend`. - Runtime base image: `python:3.14-slim`. -- Runtime process: `uvicorn backend.main:app --host 0.0.0.0 --port 8000`. +- Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`. - Exposed runtime port: `8000`. ### Required Runtime Dependencies @@ -68,7 +68,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth. ### Backend Runtime Checklist - [ ] Runtime image built from `Dockerfile.backend`. -- [ ] Runtime process is uvicorn serving `backend.main:app` on `0.0.0.0:8000`. +- [ ] Runtime process is uvicorn serving `backend.main:app` with app dir `/app/src` on `0.0.0.0:8000`. - [ ] `DATABASE_URL` is set in deployment runtime. - [ ] `GET /health` behavior matches contract. - [ ] Startup fails on runtime policy mismatch. -- 2.49.1 From 818c3d43db000d601c5bc62bb4ea32b137e73fe6 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 4 Jul 2026 10:57:29 -0400 Subject: [PATCH 07/23] fix(ci): reduce yarn install memory in source checks --- .gitea/workflows/cicd-source-checks.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 953a1ac..1438aec 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -225,11 +225,14 @@ jobs: run: | set -euo pipefail export PATH="$HOME/.local/bin:$PATH" + export UV_LINK_MODE=copy + export NODE_OPTIONS="--max-old-space-size=512" cd backend uv sync --dev cd ../frontend - yarn install --immutable || yarn install + # Keep Yarn in low-memory mode on constrained ARM runners. + yarn install --immutable --mode=skip-build --network-concurrency=1 - name: Run pre-commit source checks env: -- 2.49.1 From 07f048f57a6273126e87a434ea698678e9483fb8 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 4 Jul 2026 11:26:45 -0400 Subject: [PATCH 08/23] fix(ci): use env-based yarn concurrency tuning --- .gitea/workflows/cicd-source-checks.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 1438aec..55168e0 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -227,12 +227,13 @@ jobs: export PATH="$HOME/.local/bin:$PATH" export UV_LINK_MODE=copy export NODE_OPTIONS="--max-old-space-size=512" + export YARN_NETWORK_CONCURRENCY=1 cd backend uv sync --dev cd ../frontend # Keep Yarn in low-memory mode on constrained ARM runners. - yarn install --immutable --mode=skip-build --network-concurrency=1 + yarn install --immutable --mode=skip-build - name: Run pre-commit source checks env: -- 2.49.1 From 32d5e7d4e7d1fc118ae293d384fd495b443ca0b7 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 4 Jul 2026 15:42:32 -0400 Subject: [PATCH 09/23] fix(ci): split source checks by toolchain --- .gitea/workflows/cicd-source-checks.yaml | 126 ++++++++++++++++++++--- 1 file changed, 111 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 55168e0..9040aab 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -128,8 +128,8 @@ jobs: echo "=== Kernel Tail ===" dmesg | tail -n 120 || true - source-precommit-checks: - name: Source Pre-commit Checks (backend+frontend) + source-precommit-checks-backend: + name: Source Pre-commit Checks (backend) runs-on: ubuntu-act timeout-minutes: 40 needs: setup @@ -194,7 +194,106 @@ jobs: exit 1 } - - name: Bootstrap source-check toolchain + - name: Bootstrap backend toolchain + run: | + set -euo pipefail + + if ! command -v curl >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + + if ! command -v uv >/dev/null 2>&1; then + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + export PATH="$HOME/.local/bin:$PATH" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install backend dependencies + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_LINK_MODE=copy + + cd backend + uv sync --dev + + - name: Run backend pre-commit source checks + env: + CI: "true" + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + cd backend + uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml \ + trailing-whitespace end-of-file-fixer check-merge-conflict check-added-large-files \ + check-yaml check-json check-toml mixed-line-ending markdownlint pretty-format-toml \ + ruff ruff-format pyright pydoclint no-docstring-types + + - *failure_diagnostics_step + + source-precommit-checks-frontend: + name: Source Pre-commit Checks (frontend) + runs-on: ubuntu-act + timeout-minutes: 40 + needs: setup + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Configure registry host resolution + run: | + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + - name: Checkout source snapshot + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + run: | + set -e + umask 022 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- . + echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" + else + git checkout HEAD -- . + echo "Falling back to default branch HEAD for source checks checkout" + fi + + test -f .pre-commit-config.yaml || { + echo "❌ Missing .pre-commit-config.yaml after checkout" + exit 1 + } + + test -f frontend/package.json || { + echo "❌ Missing frontend/package.json after checkout" + find . -maxdepth 3 -type f | sort | head -n 80 + exit 1 + } + + - name: Bootstrap frontend toolchain run: | set -euo pipefail @@ -203,14 +302,12 @@ jobs: apt-get install -y -qq curl ca-certificates fi - # Install uv if missing and expose it for subsequent steps. if ! command -v uv >/dev/null 2>&1; then curl -LsSf https://astral.sh/uv/install.sh | sh fi export PATH="$HOME/.local/bin:$PATH" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - # Ensure node/corepack is available for frontend and pre-commit hooks. if ! command -v node >/dev/null 2>&1; then apt-get update -qq apt-get install -y -qq nodejs npm @@ -221,28 +318,27 @@ jobs: corepack enable - - name: Install backend and frontend dependencies + - name: Install frontend dependencies run: | set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - export UV_LINK_MODE=copy export NODE_OPTIONS="--max-old-space-size=512" export YARN_NETWORK_CONCURRENCY=1 + export PATH="$HOME/.local/bin:$PATH" cd backend - uv sync --dev - cd ../frontend - # Keep Yarn in low-memory mode on constrained ARM runners. + uv tool install pre-commit + cd frontend yarn install --immutable --mode=skip-build - - name: Run pre-commit source checks + - name: Run frontend pre-commit source checks env: CI: "true" run: | set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" + export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$HOME/.local/share/uv/tools/pre-commit/bin:$PATH" cd backend - uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml + pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml \ + eslint prettier typescript-check tsdoc-lint - *failure_diagnostics_step @@ -250,7 +346,7 @@ jobs: name: Dispatch Downstream Build runs-on: ubuntu-act timeout-minutes: 10 - needs: [setup, source-precommit-checks] + needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend] steps: - name: Identify runner run: | -- 2.49.1 From 38741c580c5dee1925029ed2c2742de187faf312 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sat, 4 Jul 2026 23:06:39 -0400 Subject: [PATCH 10/23] fix(ci): correct split source-check commands --- .gitea/workflows/cicd-source-checks.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 9040aab..6b94146 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -221,14 +221,12 @@ jobs: - name: Run backend pre-commit source checks env: CI: "true" + SKIP: "eslint,prettier,typescript-check,tsdoc-lint" run: | set -euo pipefail export PATH="$HOME/.local/bin:$PATH" cd backend - uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml \ - trailing-whitespace end-of-file-fixer check-merge-conflict check-added-large-files \ - check-yaml check-json check-toml mixed-line-ending markdownlint pretty-format-toml \ - ruff ruff-format pyright pydoclint no-docstring-types + uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml - *failure_diagnostics_step @@ -323,10 +321,6 @@ jobs: set -euo pipefail export NODE_OPTIONS="--max-old-space-size=512" export YARN_NETWORK_CONCURRENCY=1 - export PATH="$HOME/.local/bin:$PATH" - - cd backend - uv tool install pre-commit cd frontend yarn install --immutable --mode=skip-build @@ -335,10 +329,10 @@ jobs: CI: "true" run: | set -euo pipefail - export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$HOME/.local/share/uv/tools/pre-commit/bin:$PATH" - cd backend - pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml \ - eslint prettier typescript-check tsdoc-lint + cd frontend + corepack yarn eslint . --max-warnings=0 + corepack yarn vue-tsc --noEmit + corepack yarn prettier --check . - *failure_diagnostics_step -- 2.49.1 From ad919e6c11404fe5261decb799719f06da1f0cd4 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 09:53:34 -0400 Subject: [PATCH 11/23] fix(ci): add npm fallback for frontend install OOM --- .gitea/workflows/cicd-source-checks.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 6b94146..0b2fe3e 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -322,7 +322,11 @@ jobs: export NODE_OPTIONS="--max-old-space-size=512" export YARN_NETWORK_CONCURRENCY=1 cd frontend - yarn install --immutable --mode=skip-build + if ! yarn install --immutable --mode=skip-build; then + echo "Yarn install failed; retrying with npm fallback for constrained runner memory" + rm -f package-lock.json + npm install --ignore-scripts --no-audit --no-fund --prefer-offline + fi - name: Run frontend pre-commit source checks env: -- 2.49.1 From 6611c20bbfd2d9b116d2138af1f4636c2ba849a5 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 10:59:13 -0400 Subject: [PATCH 12/23] style(frontend): apply prettier to index.html --- frontend/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/index.html b/frontend/index.html index 82de1f5..d653516 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,4 +1,4 @@ - + -- 2.49.1 From 6e2b2f5e3a1bebe4a221dc7d9c0502c51bc6989a Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 11:40:31 -0400 Subject: [PATCH 13/23] feat(runners): cap runner memory by host fraction --- .../gitea-actions/deploy-runner-config.xsh | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 9ea40fa..adc970c 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -4,6 +4,7 @@ import os import subprocess +import sys import tempfile from datetime import datetime @@ -17,6 +18,23 @@ HOSTS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +DEFAULT_MEM_FRACTION = 0.75 +MIN_RUNNER_MEM_MIB = 1024 + + +def parse_mem_fraction(argv): + fraction = DEFAULT_MEM_FRACTION + for arg in argv[1:]: + if arg.startswith("--mem-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --mem-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--mem-fraction must be > 0 and <= 1") + return fraction + def ssh(host, cmd): proc = subprocess.run( [ @@ -55,7 +73,52 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr -def build_compose(compose_text): +def upsert_service_key(compose_text, service_names, key, value): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent = len(lines[i]) - len(lines[i].lstrip(" ")) + field_indent = " " * (service_indent + 2) + + block_start = i + 1 + block_end = len(lines) + j = block_start + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + block_end = j + break + j += 1 + + key_prefix = f"{key}:" + found_idx = None + for k in range(block_start, block_end): + if lines[k].strip().startswith(key_prefix): + found_idx = k + break + + new_line = f"{field_indent}{key}: {value}" + if found_idx is not None: + lines[found_idx] = new_line + i = block_end + continue + + lines.insert(block_start, new_line) + i = block_end + 1 + + return "\n".join(lines) + "\n" + + +def build_compose(compose_text, runner_mem_limit): updated = compose_text volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" @@ -68,9 +131,31 @@ def build_compose(compose_text): if add_cfg_env not in updated and force_pull_line in updated: updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env) + updated = upsert_service_key( + updated, + {"act_runner_1", "act_runner_2", "runner1", "runner2"}, + "mem_limit", + runner_mem_limit, + ) + return updated +def host_mem_limit(host, fraction): + rc, mem_kib = ssh(host, "awk '/MemTotal:/ {print $2}' /proc/meminfo") + if rc != 0 or not mem_kib.strip().isdigit(): + raise RuntimeError(f"Could not read MemTotal on {host}") + + total_kib = int(mem_kib.strip()) + target_mib = int((total_kib / 1024.0) * fraction) + if target_mib < MIN_RUNNER_MEM_MIB: + target_mib = MIN_RUNNER_MEM_MIB + return f"{target_mib}m", target_mib + + +mem_fraction = parse_mem_fraction(sys.argv) + +print(f"Runner mem fraction: {mem_fraction:.2f}") print("Fetching canonical runner config template") rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") if rc != 0 or not template.strip(): @@ -87,12 +172,20 @@ with tempfile.TemporaryDirectory() as tmpdir: for host, root in HOSTS.items(): print(f"\n===== {host} =====") + try: + runner_mem_limit, runner_mem_mib = host_mem_limit(host, mem_fraction) + except RuntimeError as exc: + print(str(exc)) + continue + + print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") + rc, compose = ssh(host, f"cat {root}/compose.yml") if rc != 0: print("Could not read compose.yml") continue - new_compose = build_compose(compose) + new_compose = build_compose(compose, runner_mem_limit) rc, _ = ssh(host, f"mkdir -p {root}/backups") rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") @@ -120,6 +213,8 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml") print(out.strip()) + rc, out = ssh(host, f"grep -n 'mem_limit:' {root}/compose.yml") + print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") print(out.strip()) -- 2.49.1 From 8f07c3ff86522dc3c862621067d92531e9197749 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 11:41:56 -0400 Subject: [PATCH 14/23] fix(runners): scale mem cap from current runner limit --- .../gitea-actions/deploy-runner-config.xsh | 108 ++++++++++++++++-- 1 file changed, 96 insertions(+), 12 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index adc970c..2d3ee69 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -118,6 +118,60 @@ def upsert_service_key(compose_text, service_names, key, value): return "\n".join(lines) + "\n" +def parse_mem_value_to_mib(raw_value): + value = raw_value.strip().lower() + if not value: + return None + + # Compose mem_limit values are typically 4096m / 4g; inspect values are bytes. + units = { + "k": 1.0 / 1024.0, + "m": 1.0, + "g": 1024.0, + } + for suffix, factor in units.items(): + if value.endswith(suffix): + num = value[: -len(suffix)] + try: + return int(float(num) * factor) + except ValueError: + return None + + if value.isdigit(): + as_int = int(value) + # Docker inspect HostConfig.Memory reports bytes. + if as_int > 0: + return int(as_int / (1024 * 1024)) + return None + + +def find_service_key_value(compose_text, service_names, key): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent = len(lines[i]) - len(lines[i].lstrip(" ")) + j = i + 1 + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + break + if cur_stripped.startswith(f"{key}:"): + return cur_stripped.split(":", 1)[1].strip() + j += 1 + i = j + + return None + + def build_compose(compose_text, runner_mem_limit): updated = compose_text @@ -141,16 +195,38 @@ def build_compose(compose_text, runner_mem_limit): return updated -def host_mem_limit(host, fraction): +def host_total_mem_mib(host): rc, mem_kib = ssh(host, "awk '/MemTotal:/ {print $2}' /proc/meminfo") if rc != 0 or not mem_kib.strip().isdigit(): raise RuntimeError(f"Could not read MemTotal on {host}") + return int(int(mem_kib.strip()) / 1024.0) - total_kib = int(mem_kib.strip()) - target_mib = int((total_kib / 1024.0) * fraction) + +def host_mem_limit(host, fraction): + total_mib = host_total_mem_mib(host) + target_mib = int(total_mib * fraction) if target_mib < MIN_RUNNER_MEM_MIB: target_mib = MIN_RUNNER_MEM_MIB - return f"{target_mib}m", target_mib + return f"{target_mib}m", target_mib, f"host-total({total_mib}MiB)" + + +def current_runner_baseline_mib(host, compose_text): + compose_value = find_service_key_value( + compose_text, + {"act_runner_1", "act_runner_2", "runner1", "runner2"}, + "mem_limit", + ) + compose_mib = parse_mem_value_to_mib(compose_value) if compose_value else None + if compose_mib and compose_mib > 0: + return compose_mib, f"compose(mem_limit={compose_value})" + + rc, inspect_mem = ssh(host, "docker inspect gitea-act-runner-1 --format '{{.HostConfig.Memory}}'") + if rc == 0: + inspect_mib = parse_mem_value_to_mib(inspect_mem) + if inspect_mib and inspect_mib > 0: + return inspect_mib, f"inspect(HostConfig.Memory={inspect_mem.strip()})" + + return None, "none" mem_fraction = parse_mem_fraction(sys.argv) @@ -172,19 +248,27 @@ with tempfile.TemporaryDirectory() as tmpdir: for host, root in HOSTS.items(): print(f"\n===== {host} =====") - try: - runner_mem_limit, runner_mem_mib = host_mem_limit(host, mem_fraction) - except RuntimeError as exc: - print(str(exc)) - continue - - print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") - rc, compose = ssh(host, f"cat {root}/compose.yml") if rc != 0: print("Could not read compose.yml") continue + baseline_mib, baseline_source = current_runner_baseline_mib(host, compose) + if baseline_mib is None: + try: + runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit(host, mem_fraction) + except RuntimeError as exc: + print(str(exc)) + continue + else: + runner_mem_mib = int(baseline_mib * mem_fraction) + if runner_mem_mib < 1: + runner_mem_mib = 1 + runner_mem_limit = f"{runner_mem_mib}m" + + print(f"runner_mem_baseline={baseline_source}") + print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") + new_compose = build_compose(compose, runner_mem_limit) rc, _ = ssh(host, f"mkdir -p {root}/backups") -- 2.49.1 From 0a29464d42d81ef02ae10a05c6daacf48f5c702f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 12:30:46 -0400 Subject: [PATCH 15/23] fix(runners): update deploy memory without mem_limit conflict --- .../gitea-actions/deploy-runner-config.xsh | 137 +++++++++++++++--- 1 file changed, 116 insertions(+), 21 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 2d3ee69..adda065 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -73,6 +73,23 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr +def service_block_bounds(lines, start_index): + service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" ")) + block_start = start_index + 1 + block_end = len(lines) + j = block_start + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + block_end = j + break + j += 1 + return service_indent, block_start, block_end + + def upsert_service_key(compose_text, service_names, key, value): lines = compose_text.splitlines() i = 0 @@ -83,22 +100,9 @@ def upsert_service_key(compose_text, service_names, key, value): i += 1 continue - service_indent = len(lines[i]) - len(lines[i].lstrip(" ")) + service_indent, block_start, block_end = service_block_bounds(lines, i) field_indent = " " * (service_indent + 2) - block_start = i + 1 - block_end = len(lines) - j = block_start - while j < len(lines): - cur = lines[j] - cur_stripped = cur.strip() - if cur_stripped: - cur_indent = len(cur) - len(cur.lstrip(" ")) - if cur_indent <= service_indent: - block_end = j - break - j += 1 - key_prefix = f"{key}:" found_idx = None for k in range(block_start, block_end): @@ -118,6 +122,68 @@ def upsert_service_key(compose_text, service_names, key, value): return "\n".join(lines) + "\n" +def remove_service_key(compose_text, service_names, key): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + _, block_start, block_end = service_block_bounds(lines, i) + key_prefix = f"{key}:" + filtered = [] + for idx in range(block_start, block_end): + if not lines[idx].strip().startswith(key_prefix): + filtered.append(lines[idx]) + + lines = lines[:block_start] + filtered + lines[block_end:] + i = block_start + len(filtered) + + return "\n".join(lines) + "\n" + + +def upsert_service_deploy_memory(compose_text, service_names, value): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent, block_start, block_end = service_block_bounds(lines, i) + deploy_indent = " " * (service_indent + 2) + resources_indent = " " * (service_indent + 4) + limits_indent = " " * (service_indent + 6) + memory_indent = " " * (service_indent + 8) + + memory_idx = None + for idx in range(block_start, block_end): + if lines[idx].strip().startswith("memory:"): + memory_idx = idx + break + + if memory_idx is not None: + lines[memory_idx] = f"{memory_indent}memory: {value}" + i = block_end + continue + + insert_lines = [ + f"{deploy_indent}deploy:", + f"{resources_indent}resources:", + f"{limits_indent}limits:", + f"{memory_indent}memory: {value}", + ] + lines = lines[:block_start] + insert_lines + lines[block_start:] + i = block_end + len(insert_lines) + + return "\n".join(lines) + "\n" + + def parse_mem_value_to_mib(raw_value): value = raw_value.strip().lower() if not value: @@ -172,6 +238,26 @@ def find_service_key_value(compose_text, service_names, key): return None +def find_service_memory_value(compose_text, service_names): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + _, block_start, block_end = service_block_bounds(lines, i) + for idx in range(block_start, block_end): + cur = lines[idx].strip() + if cur.startswith("memory:"): + return cur.split(":", 1)[1].strip() + i = block_end + + return None + + def build_compose(compose_text, runner_mem_limit): updated = compose_text @@ -185,12 +271,9 @@ def build_compose(compose_text, runner_mem_limit): if add_cfg_env not in updated and force_pull_line in updated: updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env) - updated = upsert_service_key( - updated, - {"act_runner_1", "act_runner_2", "runner1", "runner2"}, - "mem_limit", - runner_mem_limit, - ) + runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} + updated = remove_service_key(updated, runner_services, "mem_limit") + updated = upsert_service_deploy_memory(updated, runner_services, runner_mem_limit) return updated @@ -211,6 +294,14 @@ def host_mem_limit(host, fraction): def current_runner_baseline_mib(host, compose_text): + deploy_memory_value = find_service_memory_value( + compose_text, + {"act_runner_1", "act_runner_2", "runner1", "runner2"}, + ) + deploy_mib = parse_mem_value_to_mib(deploy_memory_value) if deploy_memory_value else None + if deploy_mib and deploy_mib > 0: + return deploy_mib, f"compose(deploy.memory={deploy_memory_value})" + compose_value = find_service_key_value( compose_text, {"act_runner_1", "act_runner_2", "runner1", "runner2"}, @@ -297,10 +388,14 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml") print(out.strip()) - rc, out = ssh(host, f"grep -n 'mem_limit:' {root}/compose.yml") + rc, out = ssh(host, f"grep -n 'memory:' {root}/compose.yml") print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") + if rc != 0: + print(out.strip()) + print("Failed to apply compose changes on host") + continue print(out.strip()) rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'") -- 2.49.1 From 9ae4d6eafb41004b407a7de49a4f8e036d25ffdf Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 12:38:15 -0400 Subject: [PATCH 16/23] feat(runners): set job container memory below runner --- .../gitea-actions/deploy-runner-config.xsh | 61 ++++++++++++++++++- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index adda065..d637cd7 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -19,7 +19,9 @@ HOSTS = { } DEFAULT_MEM_FRACTION = 0.75 +DEFAULT_JOB_MEM_FRACTION = 0.9 MIN_RUNNER_MEM_MIB = 1024 +MIN_JOB_MEM_MIB = 128 def parse_mem_fraction(argv): @@ -35,6 +37,20 @@ def parse_mem_fraction(argv): raise SystemExit("--mem-fraction must be > 0 and <= 1") return fraction + +def parse_job_mem_fraction(argv): + fraction = DEFAULT_JOB_MEM_FRACTION + for arg in argv[1:]: + if arg.startswith("--job-mem-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-mem-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--job-mem-fraction must be > 0 and <= 1") + return fraction + def ssh(host, cmd): proc = subprocess.run( [ @@ -184,6 +200,25 @@ def upsert_service_deploy_memory(compose_text, service_names, value): return "\n".join(lines) + "\n" +def upsert_compose_env_var(compose_text, var_name, var_value, anchor_line): + lines = compose_text.splitlines() + new_line = f" - {var_name}={var_value}" + replaced = False + + for idx, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith(f"- {var_name}="): + lines[idx] = new_line + replaced = True + + if not replaced: + for idx, line in enumerate(lines): + if line.strip() == anchor_line.strip(): + lines.insert(idx + 1, new_line) + + return "\n".join(lines) + "\n" + + def parse_mem_value_to_mib(raw_value): value = raw_value.strip().lower() if not value: @@ -258,7 +293,7 @@ def find_service_memory_value(compose_text, service_names): return None -def build_compose(compose_text, runner_mem_limit): +def build_compose(compose_text, runner_mem_limit, job_mem_limit): updated = compose_text volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" @@ -268,7 +303,15 @@ def build_compose(compose_text, runner_mem_limit): force_pull_line = " - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}" add_cfg_env = " - CONFIG_FILE=/config.yaml" - if add_cfg_env not in updated and force_pull_line in updated: + if force_pull_line in updated: + updated = upsert_compose_env_var(updated, "CONFIG_FILE", "/config.yaml", force_pull_line) + updated = upsert_compose_env_var( + updated, + "GITEA_RUNNER_JOB_CONTAINER_OPTIONS", + f"--memory={job_mem_limit} --memory-swap={job_mem_limit}", + force_pull_line, + ) + elif add_cfg_env not in updated: updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env) runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} @@ -321,8 +364,10 @@ def current_runner_baseline_mib(host, compose_text): mem_fraction = parse_mem_fraction(sys.argv) +job_mem_fraction = parse_job_mem_fraction(sys.argv) print(f"Runner mem fraction: {mem_fraction:.2f}") +print(f"Job mem fraction: {job_mem_fraction:.2f}") print("Fetching canonical runner config template") rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") if rc != 0 or not template.strip(): @@ -360,7 +405,15 @@ with tempfile.TemporaryDirectory() as tmpdir: print(f"runner_mem_baseline={baseline_source}") print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") - new_compose = build_compose(compose, runner_mem_limit) + job_mem_mib = int(runner_mem_mib * job_mem_fraction) + if job_mem_mib < MIN_JOB_MEM_MIB: + job_mem_mib = MIN_JOB_MEM_MIB + if job_mem_mib >= runner_mem_mib and runner_mem_mib > 1: + job_mem_mib = runner_mem_mib - 1 + job_mem_limit = f"{job_mem_mib}m" + print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)") + + new_compose = build_compose(compose, runner_mem_limit, job_mem_limit) rc, _ = ssh(host, f"mkdir -p {root}/backups") rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") @@ -388,6 +441,8 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml") print(out.strip()) + rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml") + print(out.strip()) rc, out = ssh(host, f"grep -n 'memory:' {root}/compose.yml") print(out.strip()) -- 2.49.1 From 2eb94d18f3970e7113b42a11ed42299e1efef682 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 13:23:34 -0400 Subject: [PATCH 17/23] fix(runners): size limits from host memory share --- .../gitea-actions/deploy-runner-config.xsh | 74 +++++++++---------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index d637cd7..39bd2e1 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -20,6 +20,7 @@ HOSTS = { DEFAULT_MEM_FRACTION = 0.75 DEFAULT_JOB_MEM_FRACTION = 0.9 +DEFAULT_HOST_MEM_SHARE = 0.25 MIN_RUNNER_MEM_MIB = 1024 MIN_JOB_MEM_MIB = 128 @@ -51,6 +52,20 @@ def parse_job_mem_fraction(argv): raise SystemExit("--job-mem-fraction must be > 0 and <= 1") return fraction + +def parse_host_mem_share(argv): + share = DEFAULT_HOST_MEM_SHARE + for arg in argv[1:]: + if arg.startswith("--host-mem-share="): + _, value = arg.split("=", 1) + try: + share = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --host-mem-share value: {value}") from exc + if share <= 0 or share > 1: + raise SystemExit("--host-mem-share must be > 0 and <= 1") + return share + def ssh(host, cmd): proc = subprocess.run( [ @@ -328,44 +343,24 @@ def host_total_mem_mib(host): return int(int(mem_kib.strip()) / 1024.0) -def host_mem_limit(host, fraction): +def host_mem_limit(host, host_mem_share, fraction): total_mib = host_total_mem_mib(host) - target_mib = int(total_mib * fraction) + host_share_mib = int(total_mib * host_mem_share) + target_mib = int(host_share_mib * fraction) if target_mib < MIN_RUNNER_MEM_MIB: target_mib = MIN_RUNNER_MEM_MIB - return f"{target_mib}m", target_mib, f"host-total({total_mib}MiB)" - - -def current_runner_baseline_mib(host, compose_text): - deploy_memory_value = find_service_memory_value( - compose_text, - {"act_runner_1", "act_runner_2", "runner1", "runner2"}, + return ( + f"{target_mib}m", + target_mib, + f"host-total({total_mib}MiB) * host-share({host_mem_share:.2f}) -> {host_share_mib}MiB * mem-fraction({fraction:.2f})", ) - deploy_mib = parse_mem_value_to_mib(deploy_memory_value) if deploy_memory_value else None - if deploy_mib and deploy_mib > 0: - return deploy_mib, f"compose(deploy.memory={deploy_memory_value})" - - compose_value = find_service_key_value( - compose_text, - {"act_runner_1", "act_runner_2", "runner1", "runner2"}, - "mem_limit", - ) - compose_mib = parse_mem_value_to_mib(compose_value) if compose_value else None - if compose_mib and compose_mib > 0: - return compose_mib, f"compose(mem_limit={compose_value})" - - rc, inspect_mem = ssh(host, "docker inspect gitea-act-runner-1 --format '{{.HostConfig.Memory}}'") - if rc == 0: - inspect_mib = parse_mem_value_to_mib(inspect_mem) - if inspect_mib and inspect_mib > 0: - return inspect_mib, f"inspect(HostConfig.Memory={inspect_mem.strip()})" - - return None, "none" mem_fraction = parse_mem_fraction(sys.argv) job_mem_fraction = parse_job_mem_fraction(sys.argv) +host_mem_share = parse_host_mem_share(sys.argv) +print(f"Host mem share: {host_mem_share:.2f}") print(f"Runner mem fraction: {mem_fraction:.2f}") print(f"Job mem fraction: {job_mem_fraction:.2f}") print("Fetching canonical runner config template") @@ -389,18 +384,15 @@ with tempfile.TemporaryDirectory() as tmpdir: print("Could not read compose.yml") continue - baseline_mib, baseline_source = current_runner_baseline_mib(host, compose) - if baseline_mib is None: - try: - runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit(host, mem_fraction) - except RuntimeError as exc: - print(str(exc)) - continue - else: - runner_mem_mib = int(baseline_mib * mem_fraction) - if runner_mem_mib < 1: - runner_mem_mib = 1 - runner_mem_limit = f"{runner_mem_mib}m" + try: + runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit( + host, + host_mem_share, + mem_fraction, + ) + except RuntimeError as exc: + print(str(exc)) + continue print(f"runner_mem_baseline={baseline_source}") print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") -- 2.49.1 From 92549b8edde41e8c8f8f46a9dcbe786b91cac19c Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 13:41:45 -0400 Subject: [PATCH 18/23] feat(runners): add host-share CPU limits --- .../gitea-actions/deploy-runner-config.xsh | 146 ++++++++++++++++-- 1 file changed, 131 insertions(+), 15 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 39bd2e1..95fc691 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -21,6 +21,9 @@ HOSTS = { DEFAULT_MEM_FRACTION = 0.75 DEFAULT_JOB_MEM_FRACTION = 0.9 DEFAULT_HOST_MEM_SHARE = 0.25 +DEFAULT_HOST_CPU_SHARE = 0.25 +DEFAULT_CPU_FRACTION = 1.0 +DEFAULT_JOB_CPU_FRACTION = 0.9 MIN_RUNNER_MEM_MIB = 1024 MIN_JOB_MEM_MIB = 128 @@ -66,6 +69,48 @@ def parse_host_mem_share(argv): raise SystemExit("--host-mem-share must be > 0 and <= 1") return share + +def parse_host_cpu_share(argv): + share = DEFAULT_HOST_CPU_SHARE + for arg in argv[1:]: + if arg.startswith("--host-cpu-share="): + _, value = arg.split("=", 1) + try: + share = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --host-cpu-share value: {value}") from exc + if share <= 0 or share > 1: + raise SystemExit("--host-cpu-share must be > 0 and <= 1") + return share + + +def parse_cpu_fraction(argv): + fraction = DEFAULT_CPU_FRACTION + for arg in argv[1:]: + if arg.startswith("--cpu-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --cpu-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--cpu-fraction must be > 0 and <= 1") + return fraction + + +def parse_job_cpu_fraction(argv): + fraction = DEFAULT_JOB_CPU_FRACTION + for arg in argv[1:]: + if arg.startswith("--job-cpu-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-cpu-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--job-cpu-fraction must be > 0 and <= 1") + return fraction + def ssh(host, cmd): proc = subprocess.run( [ @@ -176,7 +221,7 @@ def remove_service_key(compose_text, service_names, key): return "\n".join(lines) + "\n" -def upsert_service_deploy_memory(compose_text, service_names, value): +def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value): lines = compose_text.splitlines() i = 0 @@ -193,24 +238,41 @@ def upsert_service_deploy_memory(compose_text, service_names, value): memory_indent = " " * (service_indent + 8) memory_idx = None + cpus_idx = None for idx in range(block_start, block_end): if lines[idx].strip().startswith("memory:"): memory_idx = idx - break + if lines[idx].strip().startswith("cpus:"): + cpus_idx = idx if memory_idx is not None: lines[memory_idx] = f"{memory_indent}memory: {value}" + if cpus_idx is not None: + lines[cpus_idx] = f"{memory_indent}cpus: '{cpu_value}'" + + if memory_idx is not None and cpus_idx is not None: i = block_end continue - insert_lines = [ - f"{deploy_indent}deploy:", - f"{resources_indent}resources:", - f"{limits_indent}limits:", - f"{memory_indent}memory: {value}", - ] - lines = lines[:block_start] + insert_lines + lines[block_start:] - i = block_end + len(insert_lines) + if memory_idx is None and cpus_idx is None: + insert_lines = [ + f"{deploy_indent}deploy:", + f"{resources_indent}resources:", + f"{limits_indent}limits:", + f"{memory_indent}memory: {memory_value}", + f"{memory_indent}cpus: '{cpu_value}'", + ] + lines = lines[:block_start] + insert_lines + lines[block_start:] + i = block_end + len(insert_lines) + continue + + missing_lines = [] + if memory_idx is None: + missing_lines.append(f"{memory_indent}memory: {memory_value}") + if cpus_idx is None: + missing_lines.append(f"{memory_indent}cpus: '{cpu_value}'") + lines = lines[:block_start] + missing_lines + lines[block_start:] + i = block_end + len(missing_lines) return "\n".join(lines) + "\n" @@ -308,7 +370,7 @@ def find_service_memory_value(compose_text, service_names): return None -def build_compose(compose_text, runner_mem_limit, job_mem_limit): +def build_compose(compose_text, runner_mem_limit, runner_cpu_limit, job_mem_limit, job_cpu_limit): updated = compose_text volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" @@ -323,7 +385,7 @@ def build_compose(compose_text, runner_mem_limit, job_mem_limit): updated = upsert_compose_env_var( updated, "GITEA_RUNNER_JOB_CONTAINER_OPTIONS", - f"--memory={job_mem_limit} --memory-swap={job_mem_limit}", + f"--memory={job_mem_limit} --memory-swap={job_mem_limit} --cpus={job_cpu_limit}", force_pull_line, ) elif add_cfg_env not in updated: @@ -331,7 +393,7 @@ def build_compose(compose_text, runner_mem_limit, job_mem_limit): runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} updated = remove_service_key(updated, runner_services, "mem_limit") - updated = upsert_service_deploy_memory(updated, runner_services, runner_mem_limit) + updated = upsert_service_deploy_limits(updated, runner_services, runner_mem_limit, runner_cpu_limit) return updated @@ -343,6 +405,20 @@ def host_total_mem_mib(host): return int(int(mem_kib.strip()) / 1024.0) +def host_total_cpus(host): + rc, cpu_count = ssh(host, "nproc") + if rc != 0: + raise RuntimeError(f"Could not read CPU count on {host}") + value = cpu_count.strip() + try: + cpus = float(value) + except ValueError as exc: + raise RuntimeError(f"Invalid CPU count on {host}: {value}") from exc + if cpus <= 0: + raise RuntimeError(f"Non-positive CPU count on {host}: {value}") + return cpus + + def host_mem_limit(host, host_mem_share, fraction): total_mib = host_total_mem_mib(host) host_share_mib = int(total_mib * host_mem_share) @@ -356,13 +432,32 @@ def host_mem_limit(host, host_mem_share, fraction): ) +def host_cpu_limit(host, host_cpu_share, fraction): + total_cpus = host_total_cpus(host) + host_share_cpus = total_cpus * host_cpu_share + target_cpus = host_share_cpus * fraction + if target_cpus <= 0.01: + target_cpus = 0.01 + return ( + f"{target_cpus:.2f}", + target_cpus, + f"host-total({total_cpus:.0f}cpu) * host-share({host_cpu_share:.2f}) -> {host_share_cpus:.2f}cpu * cpu-fraction({fraction:.2f})", + ) + + mem_fraction = parse_mem_fraction(sys.argv) job_mem_fraction = parse_job_mem_fraction(sys.argv) host_mem_share = parse_host_mem_share(sys.argv) +host_cpu_share = parse_host_cpu_share(sys.argv) +cpu_fraction = parse_cpu_fraction(sys.argv) +job_cpu_fraction = parse_job_cpu_fraction(sys.argv) print(f"Host mem share: {host_mem_share:.2f}") print(f"Runner mem fraction: {mem_fraction:.2f}") print(f"Job mem fraction: {job_mem_fraction:.2f}") +print(f"Host cpu share: {host_cpu_share:.2f}") +print(f"Runner cpu fraction: {cpu_fraction:.2f}") +print(f"Job cpu fraction: {job_cpu_fraction:.2f}") print("Fetching canonical runner config template") rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") if rc != 0 or not template.strip(): @@ -390,12 +485,19 @@ with tempfile.TemporaryDirectory() as tmpdir: host_mem_share, mem_fraction, ) + runner_cpu_limit, runner_cpu_value, cpu_source = host_cpu_limit( + host, + host_cpu_share, + cpu_fraction, + ) except RuntimeError as exc: print(str(exc)) continue print(f"runner_mem_baseline={baseline_source}") print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") + print(f"runner_cpu_baseline={cpu_source}") + print(f"runner_cpu_limit={runner_cpu_limit} cpu") job_mem_mib = int(runner_mem_mib * job_mem_fraction) if job_mem_mib < MIN_JOB_MEM_MIB: @@ -405,7 +507,21 @@ with tempfile.TemporaryDirectory() as tmpdir: job_mem_limit = f"{job_mem_mib}m" print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)") - new_compose = build_compose(compose, runner_mem_limit, job_mem_limit) + job_cpu_value = runner_cpu_value * job_cpu_fraction + if job_cpu_value <= 0.01: + job_cpu_value = 0.01 + if job_cpu_value >= runner_cpu_value and runner_cpu_value > 0.02: + job_cpu_value = runner_cpu_value - 0.01 + job_cpu_limit = f"{job_cpu_value:.2f}" + print(f"job_cpu_limit={job_cpu_limit} cpu") + + new_compose = build_compose( + compose, + runner_mem_limit, + runner_cpu_limit, + job_mem_limit, + job_cpu_limit, + ) rc, _ = ssh(host, f"mkdir -p {root}/backups") rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") @@ -435,7 +551,7 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml") print(out.strip()) - rc, out = ssh(host, f"grep -n 'memory:' {root}/compose.yml") + rc, out = ssh(host, f"grep -nE 'memory:|cpus:' {root}/compose.yml") print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") -- 2.49.1 From f98b4d6857da784da42a1094b790feeb07786b1d Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 13:50:43 -0400 Subject: [PATCH 19/23] feat(runners): add configurable stability resource limits --- .../gitea-actions/deploy-runner-config.xsh | 186 +++++++++++++++++- 1 file changed, 182 insertions(+), 4 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 95fc691..70dc7f5 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -24,7 +24,14 @@ DEFAULT_HOST_MEM_SHARE = 0.25 DEFAULT_HOST_CPU_SHARE = 0.25 DEFAULT_CPU_FRACTION = 1.0 DEFAULT_JOB_CPU_FRACTION = 0.9 -MIN_RUNNER_MEM_MIB = 1024 +DEFAULT_RUNNER_MEM_SWAP_FACTOR = 1.0 +DEFAULT_JOB_MEM_SWAP_FACTOR = 1.0 +DEFAULT_RUNNER_PIDS_LIMIT = 1024 +DEFAULT_JOB_PIDS_LIMIT = 512 +DEFAULT_JOB_NOFILE_SOFT = 2048 +DEFAULT_JOB_NOFILE_HARD = 4096 +DEFAULT_RUNNER_MAX_PARALLEL_JOBS = 1 +MIN_RUNNER_MEM_MIB = 256 MIN_JOB_MEM_MIB = 128 @@ -111,6 +118,108 @@ def parse_job_cpu_fraction(argv): raise SystemExit("--job-cpu-fraction must be > 0 and <= 1") return fraction + +def parse_runner_mem_swap_factor(argv): + factor = DEFAULT_RUNNER_MEM_SWAP_FACTOR + for arg in argv[1:]: + if arg.startswith("--runner-memory-swap-factor="): + _, value = arg.split("=", 1) + try: + factor = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-memory-swap-factor value: {value}") from exc + if factor < 1: + raise SystemExit("--runner-memory-swap-factor must be >= 1") + return factor + + +def parse_job_mem_swap_factor(argv): + factor = DEFAULT_JOB_MEM_SWAP_FACTOR + for arg in argv[1:]: + if arg.startswith("--job-memory-swap-factor="): + _, value = arg.split("=", 1) + try: + factor = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-memory-swap-factor value: {value}") from exc + if factor < 1: + raise SystemExit("--job-memory-swap-factor must be >= 1") + return factor + + +def parse_runner_pids_limit(argv): + value = DEFAULT_RUNNER_PIDS_LIMIT + for arg in argv[1:]: + if arg.startswith("--runner-pids-limit="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-pids-limit value: {raw}") from exc + if value < 64: + raise SystemExit("--runner-pids-limit must be >= 64") + return value + + +def parse_job_pids_limit(argv): + value = DEFAULT_JOB_PIDS_LIMIT + for arg in argv[1:]: + if arg.startswith("--job-pids-limit="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-pids-limit value: {raw}") from exc + if value < 32: + raise SystemExit("--job-pids-limit must be >= 32") + return value + + +def parse_job_nofile_soft(argv): + value = DEFAULT_JOB_NOFILE_SOFT + for arg in argv[1:]: + if arg.startswith("--job-ulimit-nofile-soft="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-ulimit-nofile-soft value: {raw}") from exc + if value < 256: + raise SystemExit("--job-ulimit-nofile-soft must be >= 256") + return value + + +def parse_job_nofile_hard(argv): + value = DEFAULT_JOB_NOFILE_HARD + for arg in argv[1:]: + if arg.startswith("--job-ulimit-nofile-hard="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-ulimit-nofile-hard value: {raw}") from exc + if value < 256: + raise SystemExit("--job-ulimit-nofile-hard must be >= 256") + return value + + +def parse_runner_max_parallel_jobs(argv): + value = DEFAULT_RUNNER_MAX_PARALLEL_JOBS + for arg in argv[1:]: + if arg.startswith("--runner-max-parallel-jobs="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-max-parallel-jobs value: {raw}") from exc + if value < 1: + raise SystemExit("--runner-max-parallel-jobs must be >= 1") + return value + + +def parse_dry_run(argv): + return "--dry-run" in argv[1:] + def ssh(host, cmd): proc = subprocess.run( [ @@ -370,7 +479,20 @@ def find_service_memory_value(compose_text, service_names): return None -def build_compose(compose_text, runner_mem_limit, runner_cpu_limit, job_mem_limit, job_cpu_limit): +def build_compose( + compose_text, + runner_mem_limit, + runner_memswap_limit, + runner_cpu_limit, + runner_pids_limit, + runner_max_parallel_jobs, + job_mem_limit, + job_memswap_limit, + job_cpu_limit, + job_pids_limit, + job_nofile_soft, + job_nofile_hard, +): updated = compose_text volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" @@ -382,10 +504,20 @@ def build_compose(compose_text, runner_mem_limit, runner_cpu_limit, job_mem_limi add_cfg_env = " - CONFIG_FILE=/config.yaml" if force_pull_line in updated: updated = upsert_compose_env_var(updated, "CONFIG_FILE", "/config.yaml", force_pull_line) + updated = upsert_compose_env_var( + updated, + "GITEA_RUNNER_MAX_PARALLEL_JOBS", + str(runner_max_parallel_jobs), + force_pull_line, + ) updated = upsert_compose_env_var( updated, "GITEA_RUNNER_JOB_CONTAINER_OPTIONS", - f"--memory={job_mem_limit} --memory-swap={job_mem_limit} --cpus={job_cpu_limit}", + ( + f"--memory={job_mem_limit} --memory-swap={job_memswap_limit} " + f"--cpus={job_cpu_limit} --pids-limit={job_pids_limit} " + f"--ulimit=nofile={job_nofile_soft}:{job_nofile_hard}" + ), force_pull_line, ) elif add_cfg_env not in updated: @@ -393,6 +525,8 @@ def build_compose(compose_text, runner_mem_limit, runner_cpu_limit, job_mem_limi runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} updated = remove_service_key(updated, runner_services, "mem_limit") + updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit) + updated = upsert_service_key(updated, runner_services, "pids_limit", str(runner_pids_limit)) updated = upsert_service_deploy_limits(updated, runner_services, runner_mem_limit, runner_cpu_limit) return updated @@ -451,6 +585,17 @@ host_mem_share = parse_host_mem_share(sys.argv) host_cpu_share = parse_host_cpu_share(sys.argv) cpu_fraction = parse_cpu_fraction(sys.argv) job_cpu_fraction = parse_job_cpu_fraction(sys.argv) +runner_mem_swap_factor = parse_runner_mem_swap_factor(sys.argv) +job_mem_swap_factor = parse_job_mem_swap_factor(sys.argv) +runner_pids_limit = parse_runner_pids_limit(sys.argv) +job_pids_limit = parse_job_pids_limit(sys.argv) +job_nofile_soft = parse_job_nofile_soft(sys.argv) +job_nofile_hard = parse_job_nofile_hard(sys.argv) +runner_max_parallel_jobs = parse_runner_max_parallel_jobs(sys.argv) +dry_run = parse_dry_run(sys.argv) + +if job_nofile_hard < job_nofile_soft: + raise SystemExit("--job-ulimit-nofile-hard must be >= --job-ulimit-nofile-soft") print(f"Host mem share: {host_mem_share:.2f}") print(f"Runner mem fraction: {mem_fraction:.2f}") @@ -458,6 +603,13 @@ print(f"Job mem fraction: {job_mem_fraction:.2f}") print(f"Host cpu share: {host_cpu_share:.2f}") print(f"Runner cpu fraction: {cpu_fraction:.2f}") print(f"Job cpu fraction: {job_cpu_fraction:.2f}") +print(f"Runner memory swap factor: {runner_mem_swap_factor:.2f}") +print(f"Job memory swap factor: {job_mem_swap_factor:.2f}") +print(f"Runner pids limit: {runner_pids_limit}") +print(f"Job pids limit: {job_pids_limit}") +print(f"Job nofile soft/hard: {job_nofile_soft}/{job_nofile_hard}") +print(f"Runner max parallel jobs: {runner_max_parallel_jobs}") +print(f"Dry run: {dry_run}") print("Fetching canonical runner config template") rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") if rc != 0 or not template.strip(): @@ -496,6 +648,11 @@ with tempfile.TemporaryDirectory() as tmpdir: print(f"runner_mem_baseline={baseline_source}") print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") + runner_memswap_mib = int(runner_mem_mib * runner_mem_swap_factor) + if runner_memswap_mib < runner_mem_mib: + runner_memswap_mib = runner_mem_mib + runner_memswap_limit = f"{runner_memswap_mib}m" + print(f"runner_memswap_limit={runner_memswap_limit} ({runner_memswap_mib} MiB)") print(f"runner_cpu_baseline={cpu_source}") print(f"runner_cpu_limit={runner_cpu_limit} cpu") @@ -507,6 +664,12 @@ with tempfile.TemporaryDirectory() as tmpdir: job_mem_limit = f"{job_mem_mib}m" print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)") + job_memswap_mib = int(job_mem_mib * job_mem_swap_factor) + if job_memswap_mib < job_mem_mib: + job_memswap_mib = job_mem_mib + job_memswap_limit = f"{job_memswap_mib}m" + print(f"job_memswap_limit={job_memswap_limit} ({job_memswap_mib} MiB)") + job_cpu_value = runner_cpu_value * job_cpu_fraction if job_cpu_value <= 0.01: job_cpu_value = 0.01 @@ -514,15 +677,28 @@ with tempfile.TemporaryDirectory() as tmpdir: job_cpu_value = runner_cpu_value - 0.01 job_cpu_limit = f"{job_cpu_value:.2f}" print(f"job_cpu_limit={job_cpu_limit} cpu") + print(f"job_pids_limit={job_pids_limit}") + print(f"job_nofile_soft/hard={job_nofile_soft}/{job_nofile_hard}") new_compose = build_compose( compose, runner_mem_limit, + runner_memswap_limit, runner_cpu_limit, + runner_pids_limit, + runner_max_parallel_jobs, job_mem_limit, + job_memswap_limit, job_cpu_limit, + job_pids_limit, + job_nofile_soft, + job_nofile_hard, ) + if dry_run: + print("dry-run=true; skipping upload and restart") + continue + rc, _ = ssh(host, f"mkdir -p {root}/backups") rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") rc, _ = ssh(host, f"test -f {root}/config.yaml") @@ -551,7 +727,9 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml") print(out.strip()) - rc, out = ssh(host, f"grep -nE 'memory:|cpus:' {root}/compose.yml") + rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_MAX_PARALLEL_JOBS=' {root}/compose.yml") + print(out.strip()) + rc, out = ssh(host, f"grep -nE 'memory:|cpus:|memswap_limit:|pids_limit:' {root}/compose.yml") print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") -- 2.49.1 From eb440bc10f3941e0a6db91f4b825ca161117e55c Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 13:56:04 -0400 Subject: [PATCH 20/23] fix(runners): correct deploy limits memory variable --- scripts/gitea-actions/deploy-runner-config.xsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 70dc7f5..220e242 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -355,7 +355,7 @@ def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_ cpus_idx = idx if memory_idx is not None: - lines[memory_idx] = f"{memory_indent}memory: {value}" + lines[memory_idx] = f"{memory_indent}memory: {memory_value}" if cpus_idx is not None: lines[cpus_idx] = f"{memory_indent}cpus: '{cpu_value}'" -- 2.49.1 From f2b79792e22ea329e5e2202d585443cd0c4a6b5e Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 14:19:25 -0400 Subject: [PATCH 21/23] fix(runners): avoid pids_limit and deploy pids conflict --- .../gitea-actions/deploy-runner-config.xsh | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 220e242..f7d4268 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -330,7 +330,7 @@ def remove_service_key(compose_text, service_names, key): return "\n".join(lines) + "\n" -def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value): +def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value, pids_value): lines = compose_text.splitlines() i = 0 @@ -348,28 +348,34 @@ def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_ memory_idx = None cpus_idx = None + pids_idx = None for idx in range(block_start, block_end): if lines[idx].strip().startswith("memory:"): memory_idx = idx if lines[idx].strip().startswith("cpus:"): cpus_idx = idx + if lines[idx].strip().startswith("pids:"): + pids_idx = idx if memory_idx is not None: lines[memory_idx] = f"{memory_indent}memory: {memory_value}" if cpus_idx is not None: lines[cpus_idx] = f"{memory_indent}cpus: '{cpu_value}'" + if pids_idx is not None: + lines[pids_idx] = f"{memory_indent}pids: {pids_value}" - if memory_idx is not None and cpus_idx is not None: + if memory_idx is not None and cpus_idx is not None and pids_idx is not None: i = block_end continue - if memory_idx is None and cpus_idx is None: + if memory_idx is None and cpus_idx is None and pids_idx is None: insert_lines = [ f"{deploy_indent}deploy:", f"{resources_indent}resources:", f"{limits_indent}limits:", f"{memory_indent}memory: {memory_value}", f"{memory_indent}cpus: '{cpu_value}'", + f"{memory_indent}pids: {pids_value}", ] lines = lines[:block_start] + insert_lines + lines[block_start:] i = block_end + len(insert_lines) @@ -380,6 +386,8 @@ def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_ missing_lines.append(f"{memory_indent}memory: {memory_value}") if cpus_idx is None: missing_lines.append(f"{memory_indent}cpus: '{cpu_value}'") + if pids_idx is None: + missing_lines.append(f"{memory_indent}pids: {pids_value}") lines = lines[:block_start] + missing_lines + lines[block_start:] i = block_end + len(missing_lines) @@ -525,9 +533,15 @@ def build_compose( runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} updated = remove_service_key(updated, runner_services, "mem_limit") + updated = remove_service_key(updated, runner_services, "pids_limit") updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit) - updated = upsert_service_key(updated, runner_services, "pids_limit", str(runner_pids_limit)) - updated = upsert_service_deploy_limits(updated, runner_services, runner_mem_limit, runner_cpu_limit) + updated = upsert_service_deploy_limits( + updated, + runner_services, + runner_mem_limit, + runner_cpu_limit, + str(runner_pids_limit), + ) return updated @@ -729,7 +743,7 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_MAX_PARALLEL_JOBS=' {root}/compose.yml") print(out.strip()) - rc, out = ssh(host, f"grep -nE 'memory:|cpus:|memswap_limit:|pids_limit:' {root}/compose.yml") + rc, out = ssh(host, f"grep -nE 'memory:|cpus:|pids:|memswap_limit:|pids_limit:' {root}/compose.yml") print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") -- 2.49.1 From ac0bd9d9d8e7c913b252daae83fe0b7cadd4e988 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 15:50:25 -0400 Subject: [PATCH 22/23] fix(runners): canonicalize deploy limits block generation --- .../gitea-actions/deploy-runner-config.xsh | 78 ++++++++----------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index f7d4268..8186902 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -340,56 +340,45 @@ def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_ i += 1 continue - service_indent, block_start, block_end = service_block_bounds(lines, i) + service_indent, block_start, service_block_end = service_block_bounds(lines, i) deploy_indent = " " * (service_indent + 2) resources_indent = " " * (service_indent + 4) limits_indent = " " * (service_indent + 6) - memory_indent = " " * (service_indent + 8) + value_indent = " " * (service_indent + 8) + deploy_prefix = f"{deploy_indent}deploy:" - memory_idx = None - cpus_idx = None - pids_idx = None - for idx in range(block_start, block_end): - if lines[idx].strip().startswith("memory:"): - memory_idx = idx - if lines[idx].strip().startswith("cpus:"): - cpus_idx = idx - if lines[idx].strip().startswith("pids:"): - pids_idx = idx + # Remove an existing deploy block under this service so we can rewrite it canonically. + deploy_idx = None + deploy_end = None + for idx in range(block_start, service_block_end): + if lines[idx].startswith(deploy_prefix): + deploy_idx = idx + j = idx + 1 + while j < service_block_end: + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent + 2: + break + j += 1 + deploy_end = j + break - if memory_idx is not None: - lines[memory_idx] = f"{memory_indent}memory: {memory_value}" - if cpus_idx is not None: - lines[cpus_idx] = f"{memory_indent}cpus: '{cpu_value}'" - if pids_idx is not None: - lines[pids_idx] = f"{memory_indent}pids: {pids_value}" + if deploy_idx is not None and deploy_end is not None: + lines = lines[:deploy_idx] + lines[deploy_end:] + service_block_end -= deploy_end - deploy_idx - if memory_idx is not None and cpus_idx is not None and pids_idx is not None: - i = block_end - continue - - if memory_idx is None and cpus_idx is None and pids_idx is None: - insert_lines = [ - f"{deploy_indent}deploy:", - f"{resources_indent}resources:", - f"{limits_indent}limits:", - f"{memory_indent}memory: {memory_value}", - f"{memory_indent}cpus: '{cpu_value}'", - f"{memory_indent}pids: {pids_value}", - ] - lines = lines[:block_start] + insert_lines + lines[block_start:] - i = block_end + len(insert_lines) - continue - - missing_lines = [] - if memory_idx is None: - missing_lines.append(f"{memory_indent}memory: {memory_value}") - if cpus_idx is None: - missing_lines.append(f"{memory_indent}cpus: '{cpu_value}'") - if pids_idx is None: - missing_lines.append(f"{memory_indent}pids: {pids_value}") - lines = lines[:block_start] + missing_lines + lines[block_start:] - i = block_end + len(missing_lines) + insert_lines = [ + f"{deploy_indent}deploy:", + f"{resources_indent}resources:", + f"{limits_indent}limits:", + f"{value_indent}memory: {memory_value}", + f"{value_indent}cpus: '{cpu_value}'", + f"{value_indent}pids: {pids_value}", + ] + lines = lines[:block_start] + insert_lines + lines[block_start:] + i = service_block_end + len(insert_lines) return "\n".join(lines) + "\n" @@ -534,6 +523,7 @@ def build_compose( runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} updated = remove_service_key(updated, runner_services, "mem_limit") updated = remove_service_key(updated, runner_services, "pids_limit") + updated = remove_service_key(updated, runner_services, "pids") updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit) updated = upsert_service_deploy_limits( updated, -- 2.49.1 From 3deeee0dcb53480eb5ffd7a478e8871a90d28121 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Sun, 5 Jul 2026 21:49:29 -0400 Subject: [PATCH 23/23] fix(ci): probe runtime backend from inside container --- .gitea/workflows/cicd-tests.yaml | 45 +++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index 0fe7b64..d29c6ad 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -321,6 +321,43 @@ jobs: DB_PASSWORD="plex_password" DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" + fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")' + )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null + then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 + fi + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + dump_failure_context() { echo "=== Runtime Integration Failure Context ===" docker ps -a || true @@ -417,7 +454,7 @@ jobs: exit 1 fi - health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" if [ "${health_code}" = "200" ]; then backend_ready=true break @@ -431,7 +468,7 @@ jobs: exit 1 fi - root_code="$(curl -sS -o /tmp/blackbox-root.json -w "%{http_code}" "http://127.0.0.1:18000/" || true)" + root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then echo "❌ Root endpoint validation failed" cat /tmp/blackbox-root.json 2>/dev/null || true @@ -439,7 +476,7 @@ jobs: exit 1 fi - compatibility_code="$(curl -sS -o /tmp/blackbox-compatibility.json -w "%{http_code}" "http://127.0.0.1:18000/compatibility" || true)" + compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then echo "❌ Compatibility endpoint validation failed" cat /tmp/blackbox-compatibility.json 2>/dev/null || true @@ -447,7 +484,7 @@ jobs: exit 1 fi - health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then echo "❌ Health endpoint validation failed" cat /tmp/blackbox-health.json 2>/dev/null || true -- 2.49.1