From 1f6cafa1bcc3de2e6c4df85afc026ee70c455275 Mon Sep 17 00:00:00 2001 From: Xlorep DarkHelm Date: Sun, 5 Jul 2026 22:48:57 -0400 Subject: [PATCH] TASK: Replace integration lane with post-build backend runtime black-box tests (#72) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Replace the existing source-context integration lane with backend runtime black-box integration checks that run against started deployable containers. This change wires deployable backend image references (both commit tag and immutable digest) from the build workflow into the tests workflow, then validates runtime behavior over network endpoints. ## Why Integration confidence should come from testing running service artifacts, not only source-mounted or in-process execution. ## What Changed - Build workflow now: - Publishes deployable backend image tag reference and digest reference - Exposes both as job outputs - Passes both references into CICD Tests dispatch inputs - CICD Tests workflow now: - Accepts deployable backend tag and digest inputs - Propagates these through setup outputs - Replaces previous integration lane behavior with runtime black-box execution: - Starts isolated Docker network - Starts Postgres container - Starts backend container from digest-pinned deployable image - Enforces tag-to-digest consistency before running checks - Runs endpoint checks against live container: - GET / - GET /compatibility - GET /health - Captures backend/db logs and container state on failure - Cleans up containers and network via trap - Documentation updated: - Runtime contract enforcement section now includes runtime black-box integration checks - CI success summary now reflects runtime integration lane behavior ## Scope Included: - Backend runtime black-box integration replacement for the existing integration lane - Digest + tag identity enforcement - Failure diagnostics for triage Out of scope: - Frontend runtime smoke checks - E2E lane redesign ## Acceptance Criteria Mapping - Integration tests execute against runtime container endpoints: ✅ - Integration lane consumes built image references (not source-mounted execution): ✅ - Failures surface service logs and test logs for triage: ✅ ## Verification - Workflow files pass local validation checks - Pre-commit hooks pass on committed changes - Branch pushed and ready for PR review ## Related - Issue: #61 - Dependency context: #66 Co-authored-by: copilotcoder Reviewed-on: https://dogar.darkhelm.org/DarkHelm.org/plex-playlist/pulls/72 --- .gitea/workflows/cicd-source-checks.yaml | 132 +++- .gitea/workflows/cicd-tests.yaml | 219 +++++- .gitea/workflows/docker-build-main.yaml | 44 ++ .gitea/workflows/renovate.yml | 105 ++- Dockerfile.backend | 2 +- Dockerfile.cicd | 9 +- compose.dev.yml | 4 +- docs/CICD_SUCCESS_SUMMARY.md | 4 + docs/DEPLOYABLE_RUNTIME_CONTRACT.md | 10 +- frontend/index.html | 2 +- .../gitea-actions/deploy-runner-config.xsh | 625 +++++++++++++++++- 11 files changed, 1079 insertions(+), 77 deletions(-) diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 953a1ac..0b2fe3e 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -128,8 +128,8 @@ jobs: echo "=== Kernel Tail ===" dmesg | tail -n 120 || true - source-precommit-checks: - name: Source Pre-commit Checks (backend+frontend) + source-precommit-checks-backend: + name: Source Pre-commit Checks (backend) runs-on: ubuntu-act timeout-minutes: 40 needs: setup @@ -194,7 +194,104 @@ jobs: exit 1 } - - name: Bootstrap source-check toolchain + - name: Bootstrap backend toolchain + run: | + set -euo pipefail + + if ! command -v curl >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + + if ! command -v uv >/dev/null 2>&1; then + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + export PATH="$HOME/.local/bin:$PATH" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install backend dependencies + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_LINK_MODE=copy + + cd backend + uv sync --dev + + - name: Run backend pre-commit source checks + env: + CI: "true" + SKIP: "eslint,prettier,typescript-check,tsdoc-lint" + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + cd backend + uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml + + - *failure_diagnostics_step + + source-precommit-checks-frontend: + name: Source Pre-commit Checks (frontend) + runs-on: ubuntu-act + timeout-minutes: 40 + needs: setup + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Configure registry host resolution + run: | + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + - name: Checkout source snapshot + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + run: | + set -e + umask 022 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- . + echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" + else + git checkout HEAD -- . + echo "Falling back to default branch HEAD for source checks checkout" + fi + + test -f .pre-commit-config.yaml || { + echo "❌ Missing .pre-commit-config.yaml after checkout" + exit 1 + } + + test -f frontend/package.json || { + echo "❌ Missing frontend/package.json after checkout" + find . -maxdepth 3 -type f | sort | head -n 80 + exit 1 + } + + - name: Bootstrap frontend toolchain run: | set -euo pipefail @@ -203,14 +300,12 @@ jobs: apt-get install -y -qq curl ca-certificates fi - # Install uv if missing and expose it for subsequent steps. if ! command -v uv >/dev/null 2>&1; then curl -LsSf https://astral.sh/uv/install.sh | sh fi export PATH="$HOME/.local/bin:$PATH" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - # Ensure node/corepack is available for frontend and pre-commit hooks. if ! command -v node >/dev/null 2>&1; then apt-get update -qq apt-get install -y -qq nodejs npm @@ -221,24 +316,27 @@ jobs: corepack enable - - name: Install backend and frontend dependencies + - name: Install frontend dependencies run: | set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" + export NODE_OPTIONS="--max-old-space-size=512" + export YARN_NETWORK_CONCURRENCY=1 + cd frontend + if ! yarn install --immutable --mode=skip-build; then + echo "Yarn install failed; retrying with npm fallback for constrained runner memory" + rm -f package-lock.json + npm install --ignore-scripts --no-audit --no-fund --prefer-offline + fi - cd backend - uv sync --dev - cd ../frontend - yarn install --immutable || yarn install - - - name: Run pre-commit source checks + - name: Run frontend pre-commit source checks env: CI: "true" run: | set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - cd backend - uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml + cd frontend + corepack yarn eslint . --max-warnings=0 + corepack yarn vue-tsc --noEmit + corepack yarn prettier --check . - *failure_diagnostics_step @@ -246,7 +344,7 @@ jobs: name: Dispatch Downstream Build runs-on: ubuntu-act timeout-minutes: 10 - needs: [setup, source-precommit-checks] + needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend] steps: - name: Identify runner run: | diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index de807a8..d29c6ad 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -6,6 +6,12 @@ on: head_sha: description: Commit SHA to process required: false + deployable_backend_tag_ref: + description: Deployable backend image tag reference + required: false + deployable_backend_digest_ref: + description: Deployable backend image digest reference + required: false source_workflow: description: Upstream workflow name required: false @@ -30,6 +36,8 @@ jobs: timeout-minutes: 8 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner run: | @@ -48,6 +56,8 @@ jobs: EVENT_NAME: ${{ github.event_name }} SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} REF: ${{ github.ref }} REF_NAME: ${{ github.ref_name }} @@ -61,6 +71,8 @@ jobs: echo "source_workflow=${SOURCE_WORKFLOW}" echo "head_sha_input=${HEAD_SHA_INPUT}" echo "head_sha=${RESOLVED_HEAD_SHA}" + echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" + echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" echo "ref=${REF}" echo "ref_name=${REF_NAME}" echo "head_ref=${HEAD_REF}" @@ -70,10 +82,14 @@ jobs: id: meta env: HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} run: | RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT" - &failure_diagnostics_step name: Failure diagnostics @@ -269,7 +285,7 @@ jobs: - *failure_diagnostics_step integration-tests: - name: Integration Tests + name: Runtime Black-Box Integration Tests # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 @@ -288,28 +304,201 @@ jobs: echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run integration tests + - name: Run runtime black-box integration checks env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }} run: | + set -euo pipefail set -o pipefail - LOG_FILE="$(mktemp)" - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - if [ -d 'tests/integration' ]; then - uv run pytest tests/integration/ -v --tb=short - else - echo 'No integration tests found' + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" + LOG_FILE="$(mktemp)" + NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}" + DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}" + BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" + + fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")' + )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null + then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 fi - " 2>&1 | tee "${LOG_FILE}" + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + + dump_failure_context() { + echo "=== Runtime Integration Failure Context ===" + docker ps -a || true + echo "--- Backend logs (tail 200) ---" + docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Database logs (tail 200) ---" + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Backend inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true + echo "--- Database inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true + } + + cleanup() { + docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true + docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true + docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + { + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable backend image references from dispatch inputs" + exit 1 + fi + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then + echo "❌ Tag and digest mismatch" + echo "expected=${EXPECTED_DIGEST_REF}" + echo "actual=${ACTUAL_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" + echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" + + docker network create "${NETWORK_NAME}" + + docker run -d \ + --name "${DB_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -e POSTGRES_DB=plex_playlist \ + -e POSTGRES_USER=plex_user \ + -e POSTGRES_PASSWORD="${DB_PASSWORD}" \ + postgres:16-alpine + + db_ready=false + for i in $(seq 1 30); do + if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + docker run -d \ + --name "${BACKEND_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -p 18000:8000 \ + -e DATABASE_URL="${DB_URL}" \ + -e ENVIRONMENT=production \ + "${EXPECTED_DIGEST_REF}" + + backend_ready=false + for i in $(seq 1 40); do + backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)" + if [ "${backend_running}" != "true" ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" + if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then + echo "❌ Health endpoint validation failed" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + echo "✅ Runtime black-box integration checks passed" + } 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Integration tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of integration test output ---" + echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of runtime black-box integration output ---" tail -n 200 "${LOG_FILE}" || true exit "${TEST_STATUS}" fi diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml index 5ff3303..bdf21c2 100644 --- a/.gitea/workflows/docker-build-main.yaml +++ b/.gitea/workflows/docker-build-main.yaml @@ -138,6 +138,8 @@ jobs: timeout-minutes: 60 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner @@ -251,6 +253,44 @@ jobs: --image deployable-frontend:"${HEAD_SHA}" \ --profile frontend + - name: Push deployable backend runtime image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker push "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve deployable backend digest reference" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" + - name: Build and push complete CICD image env: PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} @@ -347,6 +387,8 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} HEAD_SHA: ${{ needs.build.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }} REPO_FULL: ${{ github.repository }} HEAD_REF: ${{ github.head_ref }} REF_NAME: ${{ github.ref_name }} @@ -426,6 +468,8 @@ jobs: --head-sha "${HEAD_SHA}" --source-workflow "CICD Main Build" --trace-id "${TRACE_ID}" + --input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + --input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" ) for API_BASE in "${CANDIDATE_API_BASES[@]}"; do diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 0bc2461..0202dd4 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -215,7 +215,7 @@ jobs: cat > renovate-config.js << 'EOF' module.exports = { platform: 'gitea', - endpoint: 'https://dogar.darkhelm.org/api/v1', + endpoint: 'https://dogar.darkhelm.org', gitAuthor: 'Renovate Bot ', repositories: ['DarkHelm.org/plex-playlist'], onboarding: false, @@ -237,36 +237,25 @@ jobs: RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_CONFIG_FILE: renovate-config.js RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://dogar.darkhelm.org/api/v1 + RENOVATE_ENDPOINT: https://dogar.darkhelm.org RENOVATE_ALLOW_INSECURE_TLS: "true" LOG_LEVEL: info run: | echo "=== Running Renovate Bot ===" + TARGET_REPO="DarkHelm.org/plex-playlist" + TARGET_ORG="${TARGET_REPO%%/*}" - select_token() { - if [ -n "${RENOVATE_TOKEN_SECRET:-}" ]; then - echo "${RENOVATE_TOKEN_SECRET}" - return 0 - fi - if [ -n "${ACTIONS_TRIGGER_TOKEN:-}" ]; then - echo "${ACTIONS_TRIGGER_TOKEN}" - return 0 - fi - if [ -n "${PACKAGE_ACCESS_TOKEN:-}" ]; then - echo "${PACKAGE_ACCESS_TOKEN}" - return 0 - fi - return 1 - } - - if ! SELECTED_TOKEN="$(select_token)"; then - echo "❌ No token available for Renovate authentication" - echo "Configure one of: RENOVATE_TOKEN, ACTIONS_TRIGGER_TOKEN, PACKAGE_ACCESS_TOKEN" - exit 1 + RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" + if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then + API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" + else + API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" fi - export RENOVATE_TOKEN="${SELECTED_TOKEN}" - unset SELECTED_TOKEN + export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + echo "Renovate endpoint: ${RENOVATE_ENDPOINT}" + echo "Preflight API endpoint: ${API_ENDPOINT}" CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then @@ -275,27 +264,71 @@ jobs: export NODE_TLS_REJECT_UNAUTHORIZED=0 fi - # Validate token before starting Renovate to fail with actionable diagnostics. - AUTH_CHECK_STATUS=$(curl -sS -o /tmp/renovate-auth-check.json -w "%{http_code}" \ - ${CURL_INSECURE_FLAG} \ - -H "Authorization: token ${RENOVATE_TOKEN}" \ - "${RENOVATE_ENDPOINT}/user" || true) + select_token_with_repo_access() { + for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do + candidate_value="${!candidate_name:-}" + if [ -z "${candidate_value}" ]; then + continue + fi - if [ "${AUTH_CHECK_STATUS}" != "200" ]; then - echo "❌ Renovate token authentication preflight failed (HTTP ${AUTH_CHECK_STATUS})" - echo "Expected a personal access token with repo/issue read-write permissions." - if [ -s /tmp/renovate-auth-check.json ]; then - echo "Response body:" - cat /tmp/renovate-auth-check.json || true + USER_STATUS=$(curl -sS -o /tmp/renovate-auth-check-user.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/user" || true) + + REPO_STATUS=$(curl -sS -o /tmp/renovate-auth-check-repo.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/repos/${TARGET_REPO}" || true) + + ORG_STATUS=$(curl -sS -o /tmp/renovate-auth-check-org.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/orgs/${TARGET_ORG}" || true) + + if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then + echo "${candidate_name}:${candidate_value}" + return 0 + fi + + echo "⚠ Token candidate ${candidate_name} rejected (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})" + done + + return 1 + } + + if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then + echo "❌ No token available for Renovate authentication with repository access" + echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." + echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}" + if [ -s /tmp/renovate-auth-check-user.json ]; then + echo "Last user endpoint response body:" + cat /tmp/renovate-auth-check-user.json || true + fi + if [ -s /tmp/renovate-auth-check-repo.json ]; then + echo "Last repo endpoint response body:" + cat /tmp/renovate-auth-check-repo.json || true + fi + if [ -s /tmp/renovate-auth-check-org.json ]; then + echo "Last org endpoint response body:" + cat /tmp/renovate-auth-check-org.json || true fi exit 1 fi - echo "✓ Renovate auth preflight passed" + SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" + SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" + export RENOVATE_TOKEN="${SELECTED_TOKEN}" + unset SELECTED_TOKEN RESULT_TOKEN + + echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then + export RENOVATE_DRY_RUN="full" echo "🔍 Running in DRY-RUN mode (no changes will be made)" + else + unset RENOVATE_DRY_RUN fi renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist diff --git a/Dockerfile.backend b/Dockerfile.backend index 1bec094..fdc12fa 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -34,4 +34,4 @@ COPY backend/ . EXPOSE 8000 # Default command - can be overridden in compose for development -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Dockerfile.cicd b/Dockerfile.cicd index babe34b..8307b83 100644 --- a/Dockerfile.cicd +++ b/Dockerfile.cicd @@ -55,6 +55,7 @@ RUN --mount=type=secret,id=ssh_private_key \ # Extract only dependency files for caching optimization mkdir -p /workspace/backend /workspace/frontend && \ cp /tmp/repo/backend/pyproject.toml /workspace/backend/ 2>/dev/null || echo "No backend pyproject.toml" && \ + cp /tmp/repo/backend/uv.lock /workspace/backend/ 2>/dev/null || echo "No backend uv.lock" && \ cp /tmp/repo/frontend/package.json /workspace/frontend/ 2>/dev/null || echo "No frontend package.json" && \ cp /tmp/repo/frontend/yarn.lock /workspace/frontend/ 2>/dev/null || echo "No frontend yarn.lock" && \ cp /tmp/repo/frontend/.yarnrc.yml /workspace/frontend/ 2>/dev/null || echo "No frontend .yarnrc.yml" && \ @@ -90,7 +91,13 @@ RUN echo "=== Installing Backend Dependencies (Phase 1: Optimized Caching) ===" echo "# Temporary README for dependency caching phase" > ../README.md && \ echo "# Minimal __init__.py for build" > src/backend/__init__.py && \ # Install all dependencies including dev dependencies - uv sync --dev && \ + if [ -f "uv.lock" ]; then \ + echo "Using locked backend dependencies (uv.lock)" && \ + uv sync --dev --frozen; \ + else \ + echo "uv.lock not found, resolving dependencies from pyproject.toml" && \ + uv sync --dev; \ + fi && \ echo "✓ Backend dependencies installed and cached"; \ else \ echo "No pyproject.toml found, skipping dependency installation"; \ diff --git a/compose.dev.yml b/compose.dev.yml index 8af76a7..2ffb871 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -22,7 +22,9 @@ services: - "8001:8000" command: - "uvicorn" - - "main:app" + - "backend.main:app" + - "--app-dir" + - "/app/src" - "--host" - "0.0.0.0" - "--port" diff --git a/docs/CICD_SUCCESS_SUMMARY.md b/docs/CICD_SUCCESS_SUMMARY.md index de4902b..a9decc3 100644 --- a/docs/CICD_SUCCESS_SUMMARY.md +++ b/docs/CICD_SUCCESS_SUMMARY.md @@ -33,6 +33,8 @@ - **E2E Tests**: Simplified Docker approach matching other successful test patterns - **Playwright**: Chromium-only CI strategy (95%+ browser market coverage) - **Registry Operations**: Consistent approach across all test phases +- **Backend Runtime Integration**: Black-box checks run against started deployable + backend containers with commit-tag and digest pinning ## 🛠️ **Critical Issues Resolved** @@ -82,6 +84,8 @@ Frontend Environment (Yarn PnP) ↓ (state regeneration) Test Execution (all phases) ↓ (consistent Docker approach) +Runtime Black-Box Integration + ↓ (deployable backend tag+digest verification) E2E Testing (Playwright) ↓ (Chromium + network resilience) ✅ SUCCESS diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 64ddf00..0c33cd2 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -31,7 +31,7 @@ Excluded: - Container build source: `Dockerfile.backend`. - Runtime base image: `python:3.14-slim`. -- Runtime process: `uvicorn main:app --host 0.0.0.0 --port 8000`. +- Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`. - Exposed runtime port: `8000`. ### Required Runtime Dependencies @@ -68,7 +68,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth. ### Backend Runtime Checklist - [ ] Runtime image built from `Dockerfile.backend`. -- [ ] Runtime process is uvicorn serving `main:app` on `0.0.0.0:8000`. +- [ ] Runtime process is uvicorn serving `backend.main:app` with app dir `/app/src` on `0.0.0.0:8000`. - [ ] `DATABASE_URL` is set in deployment runtime. - [ ] `GET /health` behavior matches contract. - [ ] Startup fails on runtime policy mismatch. @@ -157,6 +157,12 @@ Current enforcement implemented in CI: - Workflow: `.gitea/workflows/docker-build-main.yaml` - Checks include binary presence and profile-specific package metadata probes to detect CI/development tooling leakage. +- Runtime black-box integration checks against deployed backend container: + - Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job) + - Inputs: deployable backend commit tag reference and immutable digest + reference from main build dispatch. + - Assertions: digest/tag consistency and live endpoint behavior for `/`, + `/compatibility`, and `/health` on started runtime containers. Future follow-up automation under epic #66 may expand this with: diff --git a/frontend/index.html b/frontend/index.html index 82de1f5..d653516 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,4 +1,4 @@ - + diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 9ea40fa..8186902 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -4,6 +4,7 @@ import os import subprocess +import sys import tempfile from datetime import datetime @@ -17,6 +18,208 @@ HOSTS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +DEFAULT_MEM_FRACTION = 0.75 +DEFAULT_JOB_MEM_FRACTION = 0.9 +DEFAULT_HOST_MEM_SHARE = 0.25 +DEFAULT_HOST_CPU_SHARE = 0.25 +DEFAULT_CPU_FRACTION = 1.0 +DEFAULT_JOB_CPU_FRACTION = 0.9 +DEFAULT_RUNNER_MEM_SWAP_FACTOR = 1.0 +DEFAULT_JOB_MEM_SWAP_FACTOR = 1.0 +DEFAULT_RUNNER_PIDS_LIMIT = 1024 +DEFAULT_JOB_PIDS_LIMIT = 512 +DEFAULT_JOB_NOFILE_SOFT = 2048 +DEFAULT_JOB_NOFILE_HARD = 4096 +DEFAULT_RUNNER_MAX_PARALLEL_JOBS = 1 +MIN_RUNNER_MEM_MIB = 256 +MIN_JOB_MEM_MIB = 128 + + +def parse_mem_fraction(argv): + fraction = DEFAULT_MEM_FRACTION + for arg in argv[1:]: + if arg.startswith("--mem-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --mem-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--mem-fraction must be > 0 and <= 1") + return fraction + + +def parse_job_mem_fraction(argv): + fraction = DEFAULT_JOB_MEM_FRACTION + for arg in argv[1:]: + if arg.startswith("--job-mem-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-mem-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--job-mem-fraction must be > 0 and <= 1") + return fraction + + +def parse_host_mem_share(argv): + share = DEFAULT_HOST_MEM_SHARE + for arg in argv[1:]: + if arg.startswith("--host-mem-share="): + _, value = arg.split("=", 1) + try: + share = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --host-mem-share value: {value}") from exc + if share <= 0 or share > 1: + raise SystemExit("--host-mem-share must be > 0 and <= 1") + return share + + +def parse_host_cpu_share(argv): + share = DEFAULT_HOST_CPU_SHARE + for arg in argv[1:]: + if arg.startswith("--host-cpu-share="): + _, value = arg.split("=", 1) + try: + share = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --host-cpu-share value: {value}") from exc + if share <= 0 or share > 1: + raise SystemExit("--host-cpu-share must be > 0 and <= 1") + return share + + +def parse_cpu_fraction(argv): + fraction = DEFAULT_CPU_FRACTION + for arg in argv[1:]: + if arg.startswith("--cpu-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --cpu-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--cpu-fraction must be > 0 and <= 1") + return fraction + + +def parse_job_cpu_fraction(argv): + fraction = DEFAULT_JOB_CPU_FRACTION + for arg in argv[1:]: + if arg.startswith("--job-cpu-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-cpu-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--job-cpu-fraction must be > 0 and <= 1") + return fraction + + +def parse_runner_mem_swap_factor(argv): + factor = DEFAULT_RUNNER_MEM_SWAP_FACTOR + for arg in argv[1:]: + if arg.startswith("--runner-memory-swap-factor="): + _, value = arg.split("=", 1) + try: + factor = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-memory-swap-factor value: {value}") from exc + if factor < 1: + raise SystemExit("--runner-memory-swap-factor must be >= 1") + return factor + + +def parse_job_mem_swap_factor(argv): + factor = DEFAULT_JOB_MEM_SWAP_FACTOR + for arg in argv[1:]: + if arg.startswith("--job-memory-swap-factor="): + _, value = arg.split("=", 1) + try: + factor = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-memory-swap-factor value: {value}") from exc + if factor < 1: + raise SystemExit("--job-memory-swap-factor must be >= 1") + return factor + + +def parse_runner_pids_limit(argv): + value = DEFAULT_RUNNER_PIDS_LIMIT + for arg in argv[1:]: + if arg.startswith("--runner-pids-limit="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-pids-limit value: {raw}") from exc + if value < 64: + raise SystemExit("--runner-pids-limit must be >= 64") + return value + + +def parse_job_pids_limit(argv): + value = DEFAULT_JOB_PIDS_LIMIT + for arg in argv[1:]: + if arg.startswith("--job-pids-limit="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-pids-limit value: {raw}") from exc + if value < 32: + raise SystemExit("--job-pids-limit must be >= 32") + return value + + +def parse_job_nofile_soft(argv): + value = DEFAULT_JOB_NOFILE_SOFT + for arg in argv[1:]: + if arg.startswith("--job-ulimit-nofile-soft="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-ulimit-nofile-soft value: {raw}") from exc + if value < 256: + raise SystemExit("--job-ulimit-nofile-soft must be >= 256") + return value + + +def parse_job_nofile_hard(argv): + value = DEFAULT_JOB_NOFILE_HARD + for arg in argv[1:]: + if arg.startswith("--job-ulimit-nofile-hard="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-ulimit-nofile-hard value: {raw}") from exc + if value < 256: + raise SystemExit("--job-ulimit-nofile-hard must be >= 256") + return value + + +def parse_runner_max_parallel_jobs(argv): + value = DEFAULT_RUNNER_MAX_PARALLEL_JOBS + for arg in argv[1:]: + if arg.startswith("--runner-max-parallel-jobs="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-max-parallel-jobs value: {raw}") from exc + if value < 1: + raise SystemExit("--runner-max-parallel-jobs must be >= 1") + return value + + +def parse_dry_run(argv): + return "--dry-run" in argv[1:] + def ssh(host, cmd): proc = subprocess.run( [ @@ -55,7 +258,238 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr -def build_compose(compose_text): +def service_block_bounds(lines, start_index): + service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" ")) + block_start = start_index + 1 + block_end = len(lines) + j = block_start + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + block_end = j + break + j += 1 + return service_indent, block_start, block_end + + +def upsert_service_key(compose_text, service_names, key, value): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent, block_start, block_end = service_block_bounds(lines, i) + field_indent = " " * (service_indent + 2) + + key_prefix = f"{key}:" + found_idx = None + for k in range(block_start, block_end): + if lines[k].strip().startswith(key_prefix): + found_idx = k + break + + new_line = f"{field_indent}{key}: {value}" + if found_idx is not None: + lines[found_idx] = new_line + i = block_end + continue + + lines.insert(block_start, new_line) + i = block_end + 1 + + return "\n".join(lines) + "\n" + + +def remove_service_key(compose_text, service_names, key): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + _, block_start, block_end = service_block_bounds(lines, i) + key_prefix = f"{key}:" + filtered = [] + for idx in range(block_start, block_end): + if not lines[idx].strip().startswith(key_prefix): + filtered.append(lines[idx]) + + lines = lines[:block_start] + filtered + lines[block_end:] + i = block_start + len(filtered) + + return "\n".join(lines) + "\n" + + +def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value, pids_value): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent, block_start, service_block_end = service_block_bounds(lines, i) + deploy_indent = " " * (service_indent + 2) + resources_indent = " " * (service_indent + 4) + limits_indent = " " * (service_indent + 6) + value_indent = " " * (service_indent + 8) + deploy_prefix = f"{deploy_indent}deploy:" + + # Remove an existing deploy block under this service so we can rewrite it canonically. + deploy_idx = None + deploy_end = None + for idx in range(block_start, service_block_end): + if lines[idx].startswith(deploy_prefix): + deploy_idx = idx + j = idx + 1 + while j < service_block_end: + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent + 2: + break + j += 1 + deploy_end = j + break + + if deploy_idx is not None and deploy_end is not None: + lines = lines[:deploy_idx] + lines[deploy_end:] + service_block_end -= deploy_end - deploy_idx + + insert_lines = [ + f"{deploy_indent}deploy:", + f"{resources_indent}resources:", + f"{limits_indent}limits:", + f"{value_indent}memory: {memory_value}", + f"{value_indent}cpus: '{cpu_value}'", + f"{value_indent}pids: {pids_value}", + ] + lines = lines[:block_start] + insert_lines + lines[block_start:] + i = service_block_end + len(insert_lines) + + return "\n".join(lines) + "\n" + + +def upsert_compose_env_var(compose_text, var_name, var_value, anchor_line): + lines = compose_text.splitlines() + new_line = f" - {var_name}={var_value}" + replaced = False + + for idx, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith(f"- {var_name}="): + lines[idx] = new_line + replaced = True + + if not replaced: + for idx, line in enumerate(lines): + if line.strip() == anchor_line.strip(): + lines.insert(idx + 1, new_line) + + return "\n".join(lines) + "\n" + + +def parse_mem_value_to_mib(raw_value): + value = raw_value.strip().lower() + if not value: + return None + + # Compose mem_limit values are typically 4096m / 4g; inspect values are bytes. + units = { + "k": 1.0 / 1024.0, + "m": 1.0, + "g": 1024.0, + } + for suffix, factor in units.items(): + if value.endswith(suffix): + num = value[: -len(suffix)] + try: + return int(float(num) * factor) + except ValueError: + return None + + if value.isdigit(): + as_int = int(value) + # Docker inspect HostConfig.Memory reports bytes. + if as_int > 0: + return int(as_int / (1024 * 1024)) + return None + + +def find_service_key_value(compose_text, service_names, key): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent = len(lines[i]) - len(lines[i].lstrip(" ")) + j = i + 1 + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + break + if cur_stripped.startswith(f"{key}:"): + return cur_stripped.split(":", 1)[1].strip() + j += 1 + i = j + + return None + + +def find_service_memory_value(compose_text, service_names): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + _, block_start, block_end = service_block_bounds(lines, i) + for idx in range(block_start, block_end): + cur = lines[idx].strip() + if cur.startswith("memory:"): + return cur.split(":", 1)[1].strip() + i = block_end + + return None + + +def build_compose( + compose_text, + runner_mem_limit, + runner_memswap_limit, + runner_cpu_limit, + runner_pids_limit, + runner_max_parallel_jobs, + job_mem_limit, + job_memswap_limit, + job_cpu_limit, + job_pids_limit, + job_nofile_soft, + job_nofile_hard, +): updated = compose_text volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" @@ -65,12 +499,121 @@ def build_compose(compose_text): force_pull_line = " - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}" add_cfg_env = " - CONFIG_FILE=/config.yaml" - if add_cfg_env not in updated and force_pull_line in updated: + if force_pull_line in updated: + updated = upsert_compose_env_var(updated, "CONFIG_FILE", "/config.yaml", force_pull_line) + updated = upsert_compose_env_var( + updated, + "GITEA_RUNNER_MAX_PARALLEL_JOBS", + str(runner_max_parallel_jobs), + force_pull_line, + ) + updated = upsert_compose_env_var( + updated, + "GITEA_RUNNER_JOB_CONTAINER_OPTIONS", + ( + f"--memory={job_mem_limit} --memory-swap={job_memswap_limit} " + f"--cpus={job_cpu_limit} --pids-limit={job_pids_limit} " + f"--ulimit=nofile={job_nofile_soft}:{job_nofile_hard}" + ), + force_pull_line, + ) + elif add_cfg_env not in updated: updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env) + runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} + updated = remove_service_key(updated, runner_services, "mem_limit") + updated = remove_service_key(updated, runner_services, "pids_limit") + updated = remove_service_key(updated, runner_services, "pids") + updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit) + updated = upsert_service_deploy_limits( + updated, + runner_services, + runner_mem_limit, + runner_cpu_limit, + str(runner_pids_limit), + ) + return updated +def host_total_mem_mib(host): + rc, mem_kib = ssh(host, "awk '/MemTotal:/ {print $2}' /proc/meminfo") + if rc != 0 or not mem_kib.strip().isdigit(): + raise RuntimeError(f"Could not read MemTotal on {host}") + return int(int(mem_kib.strip()) / 1024.0) + + +def host_total_cpus(host): + rc, cpu_count = ssh(host, "nproc") + if rc != 0: + raise RuntimeError(f"Could not read CPU count on {host}") + value = cpu_count.strip() + try: + cpus = float(value) + except ValueError as exc: + raise RuntimeError(f"Invalid CPU count on {host}: {value}") from exc + if cpus <= 0: + raise RuntimeError(f"Non-positive CPU count on {host}: {value}") + return cpus + + +def host_mem_limit(host, host_mem_share, fraction): + total_mib = host_total_mem_mib(host) + host_share_mib = int(total_mib * host_mem_share) + target_mib = int(host_share_mib * fraction) + if target_mib < MIN_RUNNER_MEM_MIB: + target_mib = MIN_RUNNER_MEM_MIB + return ( + f"{target_mib}m", + target_mib, + f"host-total({total_mib}MiB) * host-share({host_mem_share:.2f}) -> {host_share_mib}MiB * mem-fraction({fraction:.2f})", + ) + + +def host_cpu_limit(host, host_cpu_share, fraction): + total_cpus = host_total_cpus(host) + host_share_cpus = total_cpus * host_cpu_share + target_cpus = host_share_cpus * fraction + if target_cpus <= 0.01: + target_cpus = 0.01 + return ( + f"{target_cpus:.2f}", + target_cpus, + f"host-total({total_cpus:.0f}cpu) * host-share({host_cpu_share:.2f}) -> {host_share_cpus:.2f}cpu * cpu-fraction({fraction:.2f})", + ) + + +mem_fraction = parse_mem_fraction(sys.argv) +job_mem_fraction = parse_job_mem_fraction(sys.argv) +host_mem_share = parse_host_mem_share(sys.argv) +host_cpu_share = parse_host_cpu_share(sys.argv) +cpu_fraction = parse_cpu_fraction(sys.argv) +job_cpu_fraction = parse_job_cpu_fraction(sys.argv) +runner_mem_swap_factor = parse_runner_mem_swap_factor(sys.argv) +job_mem_swap_factor = parse_job_mem_swap_factor(sys.argv) +runner_pids_limit = parse_runner_pids_limit(sys.argv) +job_pids_limit = parse_job_pids_limit(sys.argv) +job_nofile_soft = parse_job_nofile_soft(sys.argv) +job_nofile_hard = parse_job_nofile_hard(sys.argv) +runner_max_parallel_jobs = parse_runner_max_parallel_jobs(sys.argv) +dry_run = parse_dry_run(sys.argv) + +if job_nofile_hard < job_nofile_soft: + raise SystemExit("--job-ulimit-nofile-hard must be >= --job-ulimit-nofile-soft") + +print(f"Host mem share: {host_mem_share:.2f}") +print(f"Runner mem fraction: {mem_fraction:.2f}") +print(f"Job mem fraction: {job_mem_fraction:.2f}") +print(f"Host cpu share: {host_cpu_share:.2f}") +print(f"Runner cpu fraction: {cpu_fraction:.2f}") +print(f"Job cpu fraction: {job_cpu_fraction:.2f}") +print(f"Runner memory swap factor: {runner_mem_swap_factor:.2f}") +print(f"Job memory swap factor: {job_mem_swap_factor:.2f}") +print(f"Runner pids limit: {runner_pids_limit}") +print(f"Job pids limit: {job_pids_limit}") +print(f"Job nofile soft/hard: {job_nofile_soft}/{job_nofile_hard}") +print(f"Runner max parallel jobs: {runner_max_parallel_jobs}") +print(f"Dry run: {dry_run}") print("Fetching canonical runner config template") rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") if rc != 0 or not template.strip(): @@ -92,7 +635,73 @@ with tempfile.TemporaryDirectory() as tmpdir: print("Could not read compose.yml") continue - new_compose = build_compose(compose) + try: + runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit( + host, + host_mem_share, + mem_fraction, + ) + runner_cpu_limit, runner_cpu_value, cpu_source = host_cpu_limit( + host, + host_cpu_share, + cpu_fraction, + ) + except RuntimeError as exc: + print(str(exc)) + continue + + print(f"runner_mem_baseline={baseline_source}") + print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") + runner_memswap_mib = int(runner_mem_mib * runner_mem_swap_factor) + if runner_memswap_mib < runner_mem_mib: + runner_memswap_mib = runner_mem_mib + runner_memswap_limit = f"{runner_memswap_mib}m" + print(f"runner_memswap_limit={runner_memswap_limit} ({runner_memswap_mib} MiB)") + print(f"runner_cpu_baseline={cpu_source}") + print(f"runner_cpu_limit={runner_cpu_limit} cpu") + + job_mem_mib = int(runner_mem_mib * job_mem_fraction) + if job_mem_mib < MIN_JOB_MEM_MIB: + job_mem_mib = MIN_JOB_MEM_MIB + if job_mem_mib >= runner_mem_mib and runner_mem_mib > 1: + job_mem_mib = runner_mem_mib - 1 + job_mem_limit = f"{job_mem_mib}m" + print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)") + + job_memswap_mib = int(job_mem_mib * job_mem_swap_factor) + if job_memswap_mib < job_mem_mib: + job_memswap_mib = job_mem_mib + job_memswap_limit = f"{job_memswap_mib}m" + print(f"job_memswap_limit={job_memswap_limit} ({job_memswap_mib} MiB)") + + job_cpu_value = runner_cpu_value * job_cpu_fraction + if job_cpu_value <= 0.01: + job_cpu_value = 0.01 + if job_cpu_value >= runner_cpu_value and runner_cpu_value > 0.02: + job_cpu_value = runner_cpu_value - 0.01 + job_cpu_limit = f"{job_cpu_value:.2f}" + print(f"job_cpu_limit={job_cpu_limit} cpu") + print(f"job_pids_limit={job_pids_limit}") + print(f"job_nofile_soft/hard={job_nofile_soft}/{job_nofile_hard}") + + new_compose = build_compose( + compose, + runner_mem_limit, + runner_memswap_limit, + runner_cpu_limit, + runner_pids_limit, + runner_max_parallel_jobs, + job_mem_limit, + job_memswap_limit, + job_cpu_limit, + job_pids_limit, + job_nofile_soft, + job_nofile_hard, + ) + + if dry_run: + print("dry-run=true; skipping upload and restart") + continue rc, _ = ssh(host, f"mkdir -p {root}/backups") rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") @@ -120,8 +729,18 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml") print(out.strip()) + rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml") + print(out.strip()) + rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_MAX_PARALLEL_JOBS=' {root}/compose.yml") + print(out.strip()) + rc, out = ssh(host, f"grep -nE 'memory:|cpus:|pids:|memswap_limit:|pids_limit:' {root}/compose.yml") + print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") + if rc != 0: + print(out.strip()) + print("Failed to apply compose changes on host") + continue print(out.strip()) rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")