diff --git a/.gitea/workflows/cicd-source-checks.yaml b/.gitea/workflows/cicd-source-checks.yaml index 953a1ac..0b2fe3e 100644 --- a/.gitea/workflows/cicd-source-checks.yaml +++ b/.gitea/workflows/cicd-source-checks.yaml @@ -128,8 +128,8 @@ jobs: echo "=== Kernel Tail ===" dmesg | tail -n 120 || true - source-precommit-checks: - name: Source Pre-commit Checks (backend+frontend) + source-precommit-checks-backend: + name: Source Pre-commit Checks (backend) runs-on: ubuntu-act timeout-minutes: 40 needs: setup @@ -194,7 +194,104 @@ jobs: exit 1 } - - name: Bootstrap source-check toolchain + - name: Bootstrap backend toolchain + run: | + set -euo pipefail + + if ! command -v curl >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + + if ! command -v uv >/dev/null 2>&1; then + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + export PATH="$HOME/.local/bin:$PATH" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install backend dependencies + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_LINK_MODE=copy + + cd backend + uv sync --dev + + - name: Run backend pre-commit source checks + env: + CI: "true" + SKIP: "eslint,prettier,typescript-check,tsdoc-lint" + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + cd backend + uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml + + - *failure_diagnostics_step + + source-precommit-checks-frontend: + name: Source Pre-commit Checks (frontend) + runs-on: ubuntu-act + timeout-minutes: 40 + needs: setup + steps: + - name: Identify runner + run: | + echo "=== Runner Identity ===" + echo "runner_name=${RUNNER_NAME:-}" + echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}" + echo "runner_hostname_env=${HOSTNAME:-unknown}" + echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)" + echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + - name: Configure registry host resolution + run: | + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + - name: Checkout source snapshot + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + run: | + set -e + umask 022 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- . + echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}" + else + git checkout HEAD -- . + echo "Falling back to default branch HEAD for source checks checkout" + fi + + test -f .pre-commit-config.yaml || { + echo "❌ Missing .pre-commit-config.yaml after checkout" + exit 1 + } + + test -f frontend/package.json || { + echo "❌ Missing frontend/package.json after checkout" + find . -maxdepth 3 -type f | sort | head -n 80 + exit 1 + } + + - name: Bootstrap frontend toolchain run: | set -euo pipefail @@ -203,14 +300,12 @@ jobs: apt-get install -y -qq curl ca-certificates fi - # Install uv if missing and expose it for subsequent steps. if ! command -v uv >/dev/null 2>&1; then curl -LsSf https://astral.sh/uv/install.sh | sh fi export PATH="$HOME/.local/bin:$PATH" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - # Ensure node/corepack is available for frontend and pre-commit hooks. if ! command -v node >/dev/null 2>&1; then apt-get update -qq apt-get install -y -qq nodejs npm @@ -221,24 +316,27 @@ jobs: corepack enable - - name: Install backend and frontend dependencies + - name: Install frontend dependencies run: | set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" + export NODE_OPTIONS="--max-old-space-size=512" + export YARN_NETWORK_CONCURRENCY=1 + cd frontend + if ! yarn install --immutable --mode=skip-build; then + echo "Yarn install failed; retrying with npm fallback for constrained runner memory" + rm -f package-lock.json + npm install --ignore-scripts --no-audit --no-fund --prefer-offline + fi - cd backend - uv sync --dev - cd ../frontend - yarn install --immutable || yarn install - - - name: Run pre-commit source checks + - name: Run frontend pre-commit source checks env: CI: "true" run: | set -euo pipefail - export PATH="$HOME/.local/bin:$PATH" - cd backend - uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml + cd frontend + corepack yarn eslint . --max-warnings=0 + corepack yarn vue-tsc --noEmit + corepack yarn prettier --check . - *failure_diagnostics_step @@ -246,7 +344,7 @@ jobs: name: Dispatch Downstream Build runs-on: ubuntu-act timeout-minutes: 10 - needs: [setup, source-precommit-checks] + needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend] steps: - name: Identify runner run: | diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index de807a8..d29c6ad 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -6,6 +6,12 @@ on: head_sha: description: Commit SHA to process required: false + deployable_backend_tag_ref: + description: Deployable backend image tag reference + required: false + deployable_backend_digest_ref: + description: Deployable backend image digest reference + required: false source_workflow: description: Upstream workflow name required: false @@ -30,6 +36,8 @@ jobs: timeout-minutes: 8 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner run: | @@ -48,6 +56,8 @@ jobs: EVENT_NAME: ${{ github.event_name }} SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} REF: ${{ github.ref }} REF_NAME: ${{ github.ref_name }} @@ -61,6 +71,8 @@ jobs: echo "source_workflow=${SOURCE_WORKFLOW}" echo "head_sha_input=${HEAD_SHA_INPUT}" echo "head_sha=${RESOLVED_HEAD_SHA}" + echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" + echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" echo "ref=${REF}" echo "ref_name=${REF_NAME}" echo "head_ref=${HEAD_REF}" @@ -70,10 +82,14 @@ jobs: id: meta env: HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} run: | RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT" - &failure_diagnostics_step name: Failure diagnostics @@ -269,7 +285,7 @@ jobs: - *failure_diagnostics_step integration-tests: - name: Integration Tests + name: Runtime Black-Box Integration Tests # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 @@ -288,28 +304,201 @@ jobs: echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run integration tests + - name: Run runtime black-box integration checks env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }} run: | + set -euo pipefail set -o pipefail - LOG_FILE="$(mktemp)" - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - if [ -d 'tests/integration' ]; then - uv run pytest tests/integration/ -v --tb=short - else - echo 'No integration tests found' + RUN_ID="${GITHUB_RUN_ID:-local}" + RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" + LOG_FILE="$(mktemp)" + NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}" + DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}" + BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" + + fetch_backend_endpoint() { + endpoint_path="$1" + output_file="$2" + probe_tmp="$(mktemp)" + + if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")' + )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null + then + : >"${output_file}" + rm -f "${probe_tmp}" + echo "000" + return 0 fi - " 2>&1 | tee "${LOG_FILE}" + + http_code="$(head -n 1 "${probe_tmp}")" + tail -n +2 "${probe_tmp}" >"${output_file}" + rm -f "${probe_tmp}" + echo "${http_code}" + } + + dump_failure_context() { + echo "=== Runtime Integration Failure Context ===" + docker ps -a || true + echo "--- Backend logs (tail 200) ---" + docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Database logs (tail 200) ---" + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Backend inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true + echo "--- Database inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true + } + + cleanup() { + docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true + docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true + docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + { + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable backend image references from dispatch inputs" + exit 1 + fi + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then + echo "❌ Tag and digest mismatch" + echo "expected=${EXPECTED_DIGEST_REF}" + echo "actual=${ACTUAL_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" + echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" + + docker network create "${NETWORK_NAME}" + + docker run -d \ + --name "${DB_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -e POSTGRES_DB=plex_playlist \ + -e POSTGRES_USER=plex_user \ + -e POSTGRES_PASSWORD="${DB_PASSWORD}" \ + postgres:16-alpine + + db_ready=false + for i in $(seq 1 30); do + if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + docker run -d \ + --name "${BACKEND_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -p 18000:8000 \ + -e DATABASE_URL="${DB_URL}" \ + -e ENVIRONMENT=production \ + "${EXPECTED_DIGEST_REF}" + + backend_ready=false + for i in $(seq 1 40); do + backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)" + if [ "${backend_running}" != "true" ]; then + echo "❌ Backend container exited before becoming healthy" + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)" + if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)" + if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then + echo "❌ Health endpoint validation failed" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + echo "✅ Runtime black-box integration checks passed" + } 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Integration tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of integration test output ---" + echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of runtime black-box integration output ---" tail -n 200 "${LOG_FILE}" || true exit "${TEST_STATUS}" fi diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml index 5ff3303..bdf21c2 100644 --- a/.gitea/workflows/docker-build-main.yaml +++ b/.gitea/workflows/docker-build-main.yaml @@ -138,6 +138,8 @@ jobs: timeout-minutes: 60 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner @@ -251,6 +253,44 @@ jobs: --image deployable-frontend:"${HEAD_SHA}" \ --profile frontend + - name: Push deployable backend runtime image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker push "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve deployable backend digest reference" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" + - name: Build and push complete CICD image env: PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} @@ -347,6 +387,8 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} HEAD_SHA: ${{ needs.build.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }} REPO_FULL: ${{ github.repository }} HEAD_REF: ${{ github.head_ref }} REF_NAME: ${{ github.ref_name }} @@ -426,6 +468,8 @@ jobs: --head-sha "${HEAD_SHA}" --source-workflow "CICD Main Build" --trace-id "${TRACE_ID}" + --input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + --input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" ) for API_BASE in "${CANDIDATE_API_BASES[@]}"; do diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 0bc2461..0202dd4 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -215,7 +215,7 @@ jobs: cat > renovate-config.js << 'EOF' module.exports = { platform: 'gitea', - endpoint: 'https://dogar.darkhelm.org/api/v1', + endpoint: 'https://dogar.darkhelm.org', gitAuthor: 'Renovate Bot ', repositories: ['DarkHelm.org/plex-playlist'], onboarding: false, @@ -237,36 +237,25 @@ jobs: RENOVATE_DRY_RUN: ${{ inputs.dry_run }} RENOVATE_CONFIG_FILE: renovate-config.js RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://dogar.darkhelm.org/api/v1 + RENOVATE_ENDPOINT: https://dogar.darkhelm.org RENOVATE_ALLOW_INSECURE_TLS: "true" LOG_LEVEL: info run: | echo "=== Running Renovate Bot ===" + TARGET_REPO="DarkHelm.org/plex-playlist" + TARGET_ORG="${TARGET_REPO%%/*}" - select_token() { - if [ -n "${RENOVATE_TOKEN_SECRET:-}" ]; then - echo "${RENOVATE_TOKEN_SECRET}" - return 0 - fi - if [ -n "${ACTIONS_TRIGGER_TOKEN:-}" ]; then - echo "${ACTIONS_TRIGGER_TOKEN}" - return 0 - fi - if [ -n "${PACKAGE_ACCESS_TOKEN:-}" ]; then - echo "${PACKAGE_ACCESS_TOKEN}" - return 0 - fi - return 1 - } - - if ! SELECTED_TOKEN="$(select_token)"; then - echo "❌ No token available for Renovate authentication" - echo "Configure one of: RENOVATE_TOKEN, ACTIONS_TRIGGER_TOKEN, PACKAGE_ACCESS_TOKEN" - exit 1 + RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}" + if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then + API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}" + else + API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1" fi - export RENOVATE_TOKEN="${SELECTED_TOKEN}" - unset SELECTED_TOKEN + export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}" + echo "Renovate endpoint: ${RENOVATE_ENDPOINT}" + echo "Preflight API endpoint: ${API_ENDPOINT}" CURL_INSECURE_FLAG="" if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then @@ -275,27 +264,71 @@ jobs: export NODE_TLS_REJECT_UNAUTHORIZED=0 fi - # Validate token before starting Renovate to fail with actionable diagnostics. - AUTH_CHECK_STATUS=$(curl -sS -o /tmp/renovate-auth-check.json -w "%{http_code}" \ - ${CURL_INSECURE_FLAG} \ - -H "Authorization: token ${RENOVATE_TOKEN}" \ - "${RENOVATE_ENDPOINT}/user" || true) + select_token_with_repo_access() { + for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do + candidate_value="${!candidate_name:-}" + if [ -z "${candidate_value}" ]; then + continue + fi - if [ "${AUTH_CHECK_STATUS}" != "200" ]; then - echo "❌ Renovate token authentication preflight failed (HTTP ${AUTH_CHECK_STATUS})" - echo "Expected a personal access token with repo/issue read-write permissions." - if [ -s /tmp/renovate-auth-check.json ]; then - echo "Response body:" - cat /tmp/renovate-auth-check.json || true + USER_STATUS=$(curl -sS -o /tmp/renovate-auth-check-user.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/user" || true) + + REPO_STATUS=$(curl -sS -o /tmp/renovate-auth-check-repo.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/repos/${TARGET_REPO}" || true) + + ORG_STATUS=$(curl -sS -o /tmp/renovate-auth-check-org.json -w "%{http_code}" \ + ${CURL_INSECURE_FLAG} \ + -H "Authorization: token ${candidate_value}" \ + "${API_ENDPOINT}/orgs/${TARGET_ORG}" || true) + + if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then + echo "${candidate_name}:${candidate_value}" + return 0 + fi + + echo "⚠ Token candidate ${candidate_name} rejected (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})" + done + + return 1 + } + + if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then + echo "❌ No token available for Renovate authentication with repository access" + echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes." + echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}" + if [ -s /tmp/renovate-auth-check-user.json ]; then + echo "Last user endpoint response body:" + cat /tmp/renovate-auth-check-user.json || true + fi + if [ -s /tmp/renovate-auth-check-repo.json ]; then + echo "Last repo endpoint response body:" + cat /tmp/renovate-auth-check-repo.json || true + fi + if [ -s /tmp/renovate-auth-check-org.json ]; then + echo "Last org endpoint response body:" + cat /tmp/renovate-auth-check-org.json || true fi exit 1 fi - echo "✓ Renovate auth preflight passed" + SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}" + SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}" + export RENOVATE_TOKEN="${SELECTED_TOKEN}" + unset SELECTED_TOKEN RESULT_TOKEN + + echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}" # Run Renovate with configuration if [ "${RENOVATE_DRY_RUN}" = "true" ]; then + export RENOVATE_DRY_RUN="full" echo "🔍 Running in DRY-RUN mode (no changes will be made)" + else + unset RENOVATE_DRY_RUN fi renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist diff --git a/Dockerfile.backend b/Dockerfile.backend index 1bec094..fdc12fa 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -34,4 +34,4 @@ COPY backend/ . EXPOSE 8000 # Default command - can be overridden in compose for development -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Dockerfile.cicd b/Dockerfile.cicd index babe34b..8307b83 100644 --- a/Dockerfile.cicd +++ b/Dockerfile.cicd @@ -55,6 +55,7 @@ RUN --mount=type=secret,id=ssh_private_key \ # Extract only dependency files for caching optimization mkdir -p /workspace/backend /workspace/frontend && \ cp /tmp/repo/backend/pyproject.toml /workspace/backend/ 2>/dev/null || echo "No backend pyproject.toml" && \ + cp /tmp/repo/backend/uv.lock /workspace/backend/ 2>/dev/null || echo "No backend uv.lock" && \ cp /tmp/repo/frontend/package.json /workspace/frontend/ 2>/dev/null || echo "No frontend package.json" && \ cp /tmp/repo/frontend/yarn.lock /workspace/frontend/ 2>/dev/null || echo "No frontend yarn.lock" && \ cp /tmp/repo/frontend/.yarnrc.yml /workspace/frontend/ 2>/dev/null || echo "No frontend .yarnrc.yml" && \ @@ -90,7 +91,13 @@ RUN echo "=== Installing Backend Dependencies (Phase 1: Optimized Caching) ===" echo "# Temporary README for dependency caching phase" > ../README.md && \ echo "# Minimal __init__.py for build" > src/backend/__init__.py && \ # Install all dependencies including dev dependencies - uv sync --dev && \ + if [ -f "uv.lock" ]; then \ + echo "Using locked backend dependencies (uv.lock)" && \ + uv sync --dev --frozen; \ + else \ + echo "uv.lock not found, resolving dependencies from pyproject.toml" && \ + uv sync --dev; \ + fi && \ echo "✓ Backend dependencies installed and cached"; \ else \ echo "No pyproject.toml found, skipping dependency installation"; \ diff --git a/compose.dev.yml b/compose.dev.yml index 8af76a7..2ffb871 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -22,7 +22,9 @@ services: - "8001:8000" command: - "uvicorn" - - "main:app" + - "backend.main:app" + - "--app-dir" + - "/app/src" - "--host" - "0.0.0.0" - "--port" diff --git a/docs/CICD_SUCCESS_SUMMARY.md b/docs/CICD_SUCCESS_SUMMARY.md index de4902b..a9decc3 100644 --- a/docs/CICD_SUCCESS_SUMMARY.md +++ b/docs/CICD_SUCCESS_SUMMARY.md @@ -33,6 +33,8 @@ - **E2E Tests**: Simplified Docker approach matching other successful test patterns - **Playwright**: Chromium-only CI strategy (95%+ browser market coverage) - **Registry Operations**: Consistent approach across all test phases +- **Backend Runtime Integration**: Black-box checks run against started deployable + backend containers with commit-tag and digest pinning ## 🛠️ **Critical Issues Resolved** @@ -82,6 +84,8 @@ Frontend Environment (Yarn PnP) ↓ (state regeneration) Test Execution (all phases) ↓ (consistent Docker approach) +Runtime Black-Box Integration + ↓ (deployable backend tag+digest verification) E2E Testing (Playwright) ↓ (Chromium + network resilience) ✅ SUCCESS diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 64ddf00..0c33cd2 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -31,7 +31,7 @@ Excluded: - Container build source: `Dockerfile.backend`. - Runtime base image: `python:3.14-slim`. -- Runtime process: `uvicorn main:app --host 0.0.0.0 --port 8000`. +- Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`. - Exposed runtime port: `8000`. ### Required Runtime Dependencies @@ -68,7 +68,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth. ### Backend Runtime Checklist - [ ] Runtime image built from `Dockerfile.backend`. -- [ ] Runtime process is uvicorn serving `main:app` on `0.0.0.0:8000`. +- [ ] Runtime process is uvicorn serving `backend.main:app` with app dir `/app/src` on `0.0.0.0:8000`. - [ ] `DATABASE_URL` is set in deployment runtime. - [ ] `GET /health` behavior matches contract. - [ ] Startup fails on runtime policy mismatch. @@ -157,6 +157,12 @@ Current enforcement implemented in CI: - Workflow: `.gitea/workflows/docker-build-main.yaml` - Checks include binary presence and profile-specific package metadata probes to detect CI/development tooling leakage. +- Runtime black-box integration checks against deployed backend container: + - Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job) + - Inputs: deployable backend commit tag reference and immutable digest + reference from main build dispatch. + - Assertions: digest/tag consistency and live endpoint behavior for `/`, + `/compatibility`, and `/health` on started runtime containers. Future follow-up automation under epic #66 may expand this with: diff --git a/frontend/index.html b/frontend/index.html index 82de1f5..d653516 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,4 +1,4 @@ - + diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 9ea40fa..8186902 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -4,6 +4,7 @@ import os import subprocess +import sys import tempfile from datetime import datetime @@ -17,6 +18,208 @@ HOSTS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +DEFAULT_MEM_FRACTION = 0.75 +DEFAULT_JOB_MEM_FRACTION = 0.9 +DEFAULT_HOST_MEM_SHARE = 0.25 +DEFAULT_HOST_CPU_SHARE = 0.25 +DEFAULT_CPU_FRACTION = 1.0 +DEFAULT_JOB_CPU_FRACTION = 0.9 +DEFAULT_RUNNER_MEM_SWAP_FACTOR = 1.0 +DEFAULT_JOB_MEM_SWAP_FACTOR = 1.0 +DEFAULT_RUNNER_PIDS_LIMIT = 1024 +DEFAULT_JOB_PIDS_LIMIT = 512 +DEFAULT_JOB_NOFILE_SOFT = 2048 +DEFAULT_JOB_NOFILE_HARD = 4096 +DEFAULT_RUNNER_MAX_PARALLEL_JOBS = 1 +MIN_RUNNER_MEM_MIB = 256 +MIN_JOB_MEM_MIB = 128 + + +def parse_mem_fraction(argv): + fraction = DEFAULT_MEM_FRACTION + for arg in argv[1:]: + if arg.startswith("--mem-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --mem-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--mem-fraction must be > 0 and <= 1") + return fraction + + +def parse_job_mem_fraction(argv): + fraction = DEFAULT_JOB_MEM_FRACTION + for arg in argv[1:]: + if arg.startswith("--job-mem-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-mem-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--job-mem-fraction must be > 0 and <= 1") + return fraction + + +def parse_host_mem_share(argv): + share = DEFAULT_HOST_MEM_SHARE + for arg in argv[1:]: + if arg.startswith("--host-mem-share="): + _, value = arg.split("=", 1) + try: + share = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --host-mem-share value: {value}") from exc + if share <= 0 or share > 1: + raise SystemExit("--host-mem-share must be > 0 and <= 1") + return share + + +def parse_host_cpu_share(argv): + share = DEFAULT_HOST_CPU_SHARE + for arg in argv[1:]: + if arg.startswith("--host-cpu-share="): + _, value = arg.split("=", 1) + try: + share = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --host-cpu-share value: {value}") from exc + if share <= 0 or share > 1: + raise SystemExit("--host-cpu-share must be > 0 and <= 1") + return share + + +def parse_cpu_fraction(argv): + fraction = DEFAULT_CPU_FRACTION + for arg in argv[1:]: + if arg.startswith("--cpu-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --cpu-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--cpu-fraction must be > 0 and <= 1") + return fraction + + +def parse_job_cpu_fraction(argv): + fraction = DEFAULT_JOB_CPU_FRACTION + for arg in argv[1:]: + if arg.startswith("--job-cpu-fraction="): + _, value = arg.split("=", 1) + try: + fraction = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-cpu-fraction value: {value}") from exc + if fraction <= 0 or fraction > 1: + raise SystemExit("--job-cpu-fraction must be > 0 and <= 1") + return fraction + + +def parse_runner_mem_swap_factor(argv): + factor = DEFAULT_RUNNER_MEM_SWAP_FACTOR + for arg in argv[1:]: + if arg.startswith("--runner-memory-swap-factor="): + _, value = arg.split("=", 1) + try: + factor = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-memory-swap-factor value: {value}") from exc + if factor < 1: + raise SystemExit("--runner-memory-swap-factor must be >= 1") + return factor + + +def parse_job_mem_swap_factor(argv): + factor = DEFAULT_JOB_MEM_SWAP_FACTOR + for arg in argv[1:]: + if arg.startswith("--job-memory-swap-factor="): + _, value = arg.split("=", 1) + try: + factor = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid --job-memory-swap-factor value: {value}") from exc + if factor < 1: + raise SystemExit("--job-memory-swap-factor must be >= 1") + return factor + + +def parse_runner_pids_limit(argv): + value = DEFAULT_RUNNER_PIDS_LIMIT + for arg in argv[1:]: + if arg.startswith("--runner-pids-limit="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-pids-limit value: {raw}") from exc + if value < 64: + raise SystemExit("--runner-pids-limit must be >= 64") + return value + + +def parse_job_pids_limit(argv): + value = DEFAULT_JOB_PIDS_LIMIT + for arg in argv[1:]: + if arg.startswith("--job-pids-limit="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-pids-limit value: {raw}") from exc + if value < 32: + raise SystemExit("--job-pids-limit must be >= 32") + return value + + +def parse_job_nofile_soft(argv): + value = DEFAULT_JOB_NOFILE_SOFT + for arg in argv[1:]: + if arg.startswith("--job-ulimit-nofile-soft="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-ulimit-nofile-soft value: {raw}") from exc + if value < 256: + raise SystemExit("--job-ulimit-nofile-soft must be >= 256") + return value + + +def parse_job_nofile_hard(argv): + value = DEFAULT_JOB_NOFILE_HARD + for arg in argv[1:]: + if arg.startswith("--job-ulimit-nofile-hard="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --job-ulimit-nofile-hard value: {raw}") from exc + if value < 256: + raise SystemExit("--job-ulimit-nofile-hard must be >= 256") + return value + + +def parse_runner_max_parallel_jobs(argv): + value = DEFAULT_RUNNER_MAX_PARALLEL_JOBS + for arg in argv[1:]: + if arg.startswith("--runner-max-parallel-jobs="): + _, raw = arg.split("=", 1) + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"Invalid --runner-max-parallel-jobs value: {raw}") from exc + if value < 1: + raise SystemExit("--runner-max-parallel-jobs must be >= 1") + return value + + +def parse_dry_run(argv): + return "--dry-run" in argv[1:] + def ssh(host, cmd): proc = subprocess.run( [ @@ -55,7 +258,238 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr -def build_compose(compose_text): +def service_block_bounds(lines, start_index): + service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" ")) + block_start = start_index + 1 + block_end = len(lines) + j = block_start + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + block_end = j + break + j += 1 + return service_indent, block_start, block_end + + +def upsert_service_key(compose_text, service_names, key, value): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent, block_start, block_end = service_block_bounds(lines, i) + field_indent = " " * (service_indent + 2) + + key_prefix = f"{key}:" + found_idx = None + for k in range(block_start, block_end): + if lines[k].strip().startswith(key_prefix): + found_idx = k + break + + new_line = f"{field_indent}{key}: {value}" + if found_idx is not None: + lines[found_idx] = new_line + i = block_end + continue + + lines.insert(block_start, new_line) + i = block_end + 1 + + return "\n".join(lines) + "\n" + + +def remove_service_key(compose_text, service_names, key): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + _, block_start, block_end = service_block_bounds(lines, i) + key_prefix = f"{key}:" + filtered = [] + for idx in range(block_start, block_end): + if not lines[idx].strip().startswith(key_prefix): + filtered.append(lines[idx]) + + lines = lines[:block_start] + filtered + lines[block_end:] + i = block_start + len(filtered) + + return "\n".join(lines) + "\n" + + +def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value, pids_value): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent, block_start, service_block_end = service_block_bounds(lines, i) + deploy_indent = " " * (service_indent + 2) + resources_indent = " " * (service_indent + 4) + limits_indent = " " * (service_indent + 6) + value_indent = " " * (service_indent + 8) + deploy_prefix = f"{deploy_indent}deploy:" + + # Remove an existing deploy block under this service so we can rewrite it canonically. + deploy_idx = None + deploy_end = None + for idx in range(block_start, service_block_end): + if lines[idx].startswith(deploy_prefix): + deploy_idx = idx + j = idx + 1 + while j < service_block_end: + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent + 2: + break + j += 1 + deploy_end = j + break + + if deploy_idx is not None and deploy_end is not None: + lines = lines[:deploy_idx] + lines[deploy_end:] + service_block_end -= deploy_end - deploy_idx + + insert_lines = [ + f"{deploy_indent}deploy:", + f"{resources_indent}resources:", + f"{limits_indent}limits:", + f"{value_indent}memory: {memory_value}", + f"{value_indent}cpus: '{cpu_value}'", + f"{value_indent}pids: {pids_value}", + ] + lines = lines[:block_start] + insert_lines + lines[block_start:] + i = service_block_end + len(insert_lines) + + return "\n".join(lines) + "\n" + + +def upsert_compose_env_var(compose_text, var_name, var_value, anchor_line): + lines = compose_text.splitlines() + new_line = f" - {var_name}={var_value}" + replaced = False + + for idx, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith(f"- {var_name}="): + lines[idx] = new_line + replaced = True + + if not replaced: + for idx, line in enumerate(lines): + if line.strip() == anchor_line.strip(): + lines.insert(idx + 1, new_line) + + return "\n".join(lines) + "\n" + + +def parse_mem_value_to_mib(raw_value): + value = raw_value.strip().lower() + if not value: + return None + + # Compose mem_limit values are typically 4096m / 4g; inspect values are bytes. + units = { + "k": 1.0 / 1024.0, + "m": 1.0, + "g": 1024.0, + } + for suffix, factor in units.items(): + if value.endswith(suffix): + num = value[: -len(suffix)] + try: + return int(float(num) * factor) + except ValueError: + return None + + if value.isdigit(): + as_int = int(value) + # Docker inspect HostConfig.Memory reports bytes. + if as_int > 0: + return int(as_int / (1024 * 1024)) + return None + + +def find_service_key_value(compose_text, service_names, key): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + service_indent = len(lines[i]) - len(lines[i].lstrip(" ")) + j = i + 1 + while j < len(lines): + cur = lines[j] + cur_stripped = cur.strip() + if cur_stripped: + cur_indent = len(cur) - len(cur.lstrip(" ")) + if cur_indent <= service_indent: + break + if cur_stripped.startswith(f"{key}:"): + return cur_stripped.split(":", 1)[1].strip() + j += 1 + i = j + + return None + + +def find_service_memory_value(compose_text, service_names): + lines = compose_text.splitlines() + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + if stripped.rstrip(":") not in service_names or not stripped.endswith(":"): + i += 1 + continue + + _, block_start, block_end = service_block_bounds(lines, i) + for idx in range(block_start, block_end): + cur = lines[idx].strip() + if cur.startswith("memory:"): + return cur.split(":", 1)[1].strip() + i = block_end + + return None + + +def build_compose( + compose_text, + runner_mem_limit, + runner_memswap_limit, + runner_cpu_limit, + runner_pids_limit, + runner_max_parallel_jobs, + job_mem_limit, + job_memswap_limit, + job_cpu_limit, + job_pids_limit, + job_nofile_soft, + job_nofile_hard, +): updated = compose_text volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" @@ -65,12 +499,121 @@ def build_compose(compose_text): force_pull_line = " - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}" add_cfg_env = " - CONFIG_FILE=/config.yaml" - if add_cfg_env not in updated and force_pull_line in updated: + if force_pull_line in updated: + updated = upsert_compose_env_var(updated, "CONFIG_FILE", "/config.yaml", force_pull_line) + updated = upsert_compose_env_var( + updated, + "GITEA_RUNNER_MAX_PARALLEL_JOBS", + str(runner_max_parallel_jobs), + force_pull_line, + ) + updated = upsert_compose_env_var( + updated, + "GITEA_RUNNER_JOB_CONTAINER_OPTIONS", + ( + f"--memory={job_mem_limit} --memory-swap={job_memswap_limit} " + f"--cpus={job_cpu_limit} --pids-limit={job_pids_limit} " + f"--ulimit=nofile={job_nofile_soft}:{job_nofile_hard}" + ), + force_pull_line, + ) + elif add_cfg_env not in updated: updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env) + runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"} + updated = remove_service_key(updated, runner_services, "mem_limit") + updated = remove_service_key(updated, runner_services, "pids_limit") + updated = remove_service_key(updated, runner_services, "pids") + updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit) + updated = upsert_service_deploy_limits( + updated, + runner_services, + runner_mem_limit, + runner_cpu_limit, + str(runner_pids_limit), + ) + return updated +def host_total_mem_mib(host): + rc, mem_kib = ssh(host, "awk '/MemTotal:/ {print $2}' /proc/meminfo") + if rc != 0 or not mem_kib.strip().isdigit(): + raise RuntimeError(f"Could not read MemTotal on {host}") + return int(int(mem_kib.strip()) / 1024.0) + + +def host_total_cpus(host): + rc, cpu_count = ssh(host, "nproc") + if rc != 0: + raise RuntimeError(f"Could not read CPU count on {host}") + value = cpu_count.strip() + try: + cpus = float(value) + except ValueError as exc: + raise RuntimeError(f"Invalid CPU count on {host}: {value}") from exc + if cpus <= 0: + raise RuntimeError(f"Non-positive CPU count on {host}: {value}") + return cpus + + +def host_mem_limit(host, host_mem_share, fraction): + total_mib = host_total_mem_mib(host) + host_share_mib = int(total_mib * host_mem_share) + target_mib = int(host_share_mib * fraction) + if target_mib < MIN_RUNNER_MEM_MIB: + target_mib = MIN_RUNNER_MEM_MIB + return ( + f"{target_mib}m", + target_mib, + f"host-total({total_mib}MiB) * host-share({host_mem_share:.2f}) -> {host_share_mib}MiB * mem-fraction({fraction:.2f})", + ) + + +def host_cpu_limit(host, host_cpu_share, fraction): + total_cpus = host_total_cpus(host) + host_share_cpus = total_cpus * host_cpu_share + target_cpus = host_share_cpus * fraction + if target_cpus <= 0.01: + target_cpus = 0.01 + return ( + f"{target_cpus:.2f}", + target_cpus, + f"host-total({total_cpus:.0f}cpu) * host-share({host_cpu_share:.2f}) -> {host_share_cpus:.2f}cpu * cpu-fraction({fraction:.2f})", + ) + + +mem_fraction = parse_mem_fraction(sys.argv) +job_mem_fraction = parse_job_mem_fraction(sys.argv) +host_mem_share = parse_host_mem_share(sys.argv) +host_cpu_share = parse_host_cpu_share(sys.argv) +cpu_fraction = parse_cpu_fraction(sys.argv) +job_cpu_fraction = parse_job_cpu_fraction(sys.argv) +runner_mem_swap_factor = parse_runner_mem_swap_factor(sys.argv) +job_mem_swap_factor = parse_job_mem_swap_factor(sys.argv) +runner_pids_limit = parse_runner_pids_limit(sys.argv) +job_pids_limit = parse_job_pids_limit(sys.argv) +job_nofile_soft = parse_job_nofile_soft(sys.argv) +job_nofile_hard = parse_job_nofile_hard(sys.argv) +runner_max_parallel_jobs = parse_runner_max_parallel_jobs(sys.argv) +dry_run = parse_dry_run(sys.argv) + +if job_nofile_hard < job_nofile_soft: + raise SystemExit("--job-ulimit-nofile-hard must be >= --job-ulimit-nofile-soft") + +print(f"Host mem share: {host_mem_share:.2f}") +print(f"Runner mem fraction: {mem_fraction:.2f}") +print(f"Job mem fraction: {job_mem_fraction:.2f}") +print(f"Host cpu share: {host_cpu_share:.2f}") +print(f"Runner cpu fraction: {cpu_fraction:.2f}") +print(f"Job cpu fraction: {job_cpu_fraction:.2f}") +print(f"Runner memory swap factor: {runner_mem_swap_factor:.2f}") +print(f"Job memory swap factor: {job_mem_swap_factor:.2f}") +print(f"Runner pids limit: {runner_pids_limit}") +print(f"Job pids limit: {job_pids_limit}") +print(f"Job nofile soft/hard: {job_nofile_soft}/{job_nofile_hard}") +print(f"Runner max parallel jobs: {runner_max_parallel_jobs}") +print(f"Dry run: {dry_run}") print("Fetching canonical runner config template") rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") if rc != 0 or not template.strip(): @@ -92,7 +635,73 @@ with tempfile.TemporaryDirectory() as tmpdir: print("Could not read compose.yml") continue - new_compose = build_compose(compose) + try: + runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit( + host, + host_mem_share, + mem_fraction, + ) + runner_cpu_limit, runner_cpu_value, cpu_source = host_cpu_limit( + host, + host_cpu_share, + cpu_fraction, + ) + except RuntimeError as exc: + print(str(exc)) + continue + + print(f"runner_mem_baseline={baseline_source}") + print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)") + runner_memswap_mib = int(runner_mem_mib * runner_mem_swap_factor) + if runner_memswap_mib < runner_mem_mib: + runner_memswap_mib = runner_mem_mib + runner_memswap_limit = f"{runner_memswap_mib}m" + print(f"runner_memswap_limit={runner_memswap_limit} ({runner_memswap_mib} MiB)") + print(f"runner_cpu_baseline={cpu_source}") + print(f"runner_cpu_limit={runner_cpu_limit} cpu") + + job_mem_mib = int(runner_mem_mib * job_mem_fraction) + if job_mem_mib < MIN_JOB_MEM_MIB: + job_mem_mib = MIN_JOB_MEM_MIB + if job_mem_mib >= runner_mem_mib and runner_mem_mib > 1: + job_mem_mib = runner_mem_mib - 1 + job_mem_limit = f"{job_mem_mib}m" + print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)") + + job_memswap_mib = int(job_mem_mib * job_mem_swap_factor) + if job_memswap_mib < job_mem_mib: + job_memswap_mib = job_mem_mib + job_memswap_limit = f"{job_memswap_mib}m" + print(f"job_memswap_limit={job_memswap_limit} ({job_memswap_mib} MiB)") + + job_cpu_value = runner_cpu_value * job_cpu_fraction + if job_cpu_value <= 0.01: + job_cpu_value = 0.01 + if job_cpu_value >= runner_cpu_value and runner_cpu_value > 0.02: + job_cpu_value = runner_cpu_value - 0.01 + job_cpu_limit = f"{job_cpu_value:.2f}" + print(f"job_cpu_limit={job_cpu_limit} cpu") + print(f"job_pids_limit={job_pids_limit}") + print(f"job_nofile_soft/hard={job_nofile_soft}/{job_nofile_hard}") + + new_compose = build_compose( + compose, + runner_mem_limit, + runner_memswap_limit, + runner_cpu_limit, + runner_pids_limit, + runner_max_parallel_jobs, + job_mem_limit, + job_memswap_limit, + job_cpu_limit, + job_pids_limit, + job_nofile_soft, + job_nofile_hard, + ) + + if dry_run: + print("dry-run=true; skipping upload and restart") + continue rc, _ = ssh(host, f"mkdir -p {root}/backups") rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") @@ -120,8 +729,18 @@ with tempfile.TemporaryDirectory() as tmpdir: print(out.strip()) rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml") print(out.strip()) + rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml") + print(out.strip()) + rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_MAX_PARALLEL_JOBS=' {root}/compose.yml") + print(out.strip()) + rc, out = ssh(host, f"grep -nE 'memory:|cpus:|pids:|memswap_limit:|pids_limit:' {root}/compose.yml") + print(out.strip()) rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") + if rc != 0: + print(out.strip()) + print("Failed to apply compose changes on host") + continue print(out.strip()) rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")