TASK: Replace integration lane with post-build backend runtime black-box tests #72

Merged
darkhelm merged 23 commits from feature/issue-61-runtime-blackbox-integration into main 2026-07-05 22:49:09 -04:00
11 changed files with 1079 additions and 77 deletions

View File

@@ -128,8 +128,8 @@ jobs:
echo "=== Kernel Tail ==="
dmesg | tail -n 120 || true
source-precommit-checks:
name: Source Pre-commit Checks (backend+frontend)
source-precommit-checks-backend:
name: Source Pre-commit Checks (backend)
runs-on: ubuntu-act
timeout-minutes: 40
needs: setup
@@ -194,7 +194,104 @@ jobs:
exit 1
}
- name: Bootstrap source-check toolchain
- name: Bootstrap backend toolchain
run: |
set -euo pipefail
if ! command -v curl >/dev/null 2>&1; then
apt-get update -qq
apt-get install -y -qq curl ca-certificates
fi
if ! command -v uv >/dev/null 2>&1; then
curl -LsSf https://astral.sh/uv/install.sh | sh
fi
export PATH="$HOME/.local/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Install backend dependencies
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$PATH"
export UV_LINK_MODE=copy
cd backend
uv sync --dev
- name: Run backend pre-commit source checks
env:
CI: "true"
SKIP: "eslint,prettier,typescript-check,tsdoc-lint"
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$PATH"
cd backend
uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml
- *failure_diagnostics_step
source-precommit-checks-frontend:
name: Source Pre-commit Checks (frontend)
runs-on: ubuntu-act
timeout-minutes: 40
needs: setup
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- name: Configure registry host resolution
run: |
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
- name: Checkout source snapshot
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
run: |
set -e
umask 022
trap 'rm -f ~/.ssh/id_rsa' EXIT
mkdir -p ~/.ssh
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
git checkout FETCH_HEAD -- .
echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}"
else
git checkout HEAD -- .
echo "Falling back to default branch HEAD for source checks checkout"
fi
test -f .pre-commit-config.yaml || {
echo "❌ Missing .pre-commit-config.yaml after checkout"
exit 1
}
test -f frontend/package.json || {
echo "❌ Missing frontend/package.json after checkout"
find . -maxdepth 3 -type f | sort | head -n 80
exit 1
}
- name: Bootstrap frontend toolchain
run: |
set -euo pipefail
@@ -203,14 +300,12 @@ jobs:
apt-get install -y -qq curl ca-certificates
fi
# Install uv if missing and expose it for subsequent steps.
if ! command -v uv >/dev/null 2>&1; then
curl -LsSf https://astral.sh/uv/install.sh | sh
fi
export PATH="$HOME/.local/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
# Ensure node/corepack is available for frontend and pre-commit hooks.
if ! command -v node >/dev/null 2>&1; then
apt-get update -qq
apt-get install -y -qq nodejs npm
@@ -221,24 +316,27 @@ jobs:
corepack enable
- name: Install backend and frontend dependencies
- name: Install frontend dependencies
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$PATH"
export NODE_OPTIONS="--max-old-space-size=512"
export YARN_NETWORK_CONCURRENCY=1
cd frontend
if ! yarn install --immutable --mode=skip-build; then
echo "Yarn install failed; retrying with npm fallback for constrained runner memory"
rm -f package-lock.json
npm install --ignore-scripts --no-audit --no-fund --prefer-offline
fi
cd backend
uv sync --dev
cd ../frontend
yarn install --immutable || yarn install
- name: Run pre-commit source checks
- name: Run frontend pre-commit source checks
env:
CI: "true"
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$PATH"
cd backend
uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml
cd frontend
corepack yarn eslint . --max-warnings=0
corepack yarn vue-tsc --noEmit
corepack yarn prettier --check .
- *failure_diagnostics_step
@@ -246,7 +344,7 @@ jobs:
name: Dispatch Downstream Build
runs-on: ubuntu-act
timeout-minutes: 10
needs: [setup, source-precommit-checks]
needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend]
steps:
- name: Identify runner
run: |

View File

@@ -6,6 +6,12 @@ on:
head_sha:
description: Commit SHA to process
required: false
deployable_backend_tag_ref:
description: Deployable backend image tag reference
required: false
deployable_backend_digest_ref:
description: Deployable backend image digest reference
required: false
source_workflow:
description: Upstream workflow name
required: false
@@ -30,6 +36,8 @@ jobs:
timeout-minutes: 8
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }}
deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }}
steps:
- name: Identify runner
run: |
@@ -48,6 +56,8 @@ jobs:
EVENT_NAME: ${{ github.event_name }}
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }}
HEAD_SHA_FALLBACK: ${{ github.sha }}
REF: ${{ github.ref }}
REF_NAME: ${{ github.ref_name }}
@@ -61,6 +71,8 @@ jobs:
echo "source_workflow=${SOURCE_WORKFLOW}"
echo "head_sha_input=${HEAD_SHA_INPUT}"
echo "head_sha=${RESOLVED_HEAD_SHA}"
echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}"
echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}"
echo "ref=${REF}"
echo "ref_name=${REF_NAME}"
echo "head_ref=${HEAD_REF}"
@@ -70,10 +82,14 @@ jobs:
id: meta
env:
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }}
HEAD_SHA_FALLBACK: ${{ github.sha }}
run: |
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT"
- &failure_diagnostics_step
name: Failure diagnostics
@@ -269,7 +285,7 @@ jobs:
- *failure_diagnostics_step
integration-tests:
name: Integration Tests
name: Runtime Black-Box Integration Tests
# Pin integration tests to high-memory worker to reduce setup-stage runner churn.
runs-on: ubuntu-act-8gb
timeout-minutes: 20
@@ -288,28 +304,201 @@ jobs:
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- *configure_registry_host_step
- *ensure_cicd_image_step
- name: Run integration tests
- name: Run runtime black-box integration checks
env:
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }}
run: |
set -euo pipefail
set -o pipefail
LOG_FILE="$(mktemp)"
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
cd /workspace/backend &&
source .venv/bin/activate &&
if [ -d 'tests/integration' ]; then
uv run pytest tests/integration/ -v --tb=short
else
echo 'No integration tests found'
RUN_ID="${GITHUB_RUN_ID:-local}"
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
LOG_FILE="$(mktemp)"
NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}"
DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}"
BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}"
DB_PASSWORD="plex_password"
DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist"
fetch_backend_endpoint() {
endpoint_path="$1"
output_file="$2"
probe_tmp="$(mktemp)"
if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \
'import sys' \
'import urllib.error' \
'import urllib.request' \
'endpoint_path = sys.argv[1]' \
'url = f"http://127.0.0.1:8000{endpoint_path}"' \
'try:' \
' with urllib.request.urlopen(url, timeout=2) as response:' \
' body = response.read().decode("utf-8", errors="replace")' \
' print(response.status)' \
' print(body)' \
'except urllib.error.HTTPError as err:' \
' body = err.read().decode("utf-8", errors="replace")' \
' print(err.code)' \
' print(body)' \
'except Exception:' \
' print("000")' \
' print("")'
)" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null
then
: >"${output_file}"
rm -f "${probe_tmp}"
echo "000"
return 0
fi
" 2>&1 | tee "${LOG_FILE}"
http_code="$(head -n 1 "${probe_tmp}")"
tail -n +2 "${probe_tmp}" >"${output_file}"
rm -f "${probe_tmp}"
echo "${http_code}"
}
dump_failure_context() {
echo "=== Runtime Integration Failure Context ==="
docker ps -a || true
echo "--- Backend logs (tail 200) ---"
docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true
echo "--- Database logs (tail 200) ---"
docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true
echo "--- Backend inspect (status/image) ---"
docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true
echo "--- Database inspect (status/image) ---"
docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true
}
cleanup() {
docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true
docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true
docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true
}
trap cleanup EXIT
{
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
echo "❌ Missing deployable backend image references from dispatch inputs"
exit 1
fi
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
docker pull "${DEPLOYABLE_BACKEND_TAG_REF}"
docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}"
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
ACTUAL_TAG_DIGEST_REF="$({
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
| head -n 1
} || true)"
if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then
echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
exit 1
fi
if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then
echo "❌ Tag and digest mismatch"
echo "expected=${EXPECTED_DIGEST_REF}"
echo "actual=${ACTUAL_TAG_DIGEST_REF}"
exit 1
fi
echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}"
echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}"
docker network create "${NETWORK_NAME}"
docker run -d \
--name "${DB_CONTAINER}" \
--network "${NETWORK_NAME}" \
-e POSTGRES_DB=plex_playlist \
-e POSTGRES_USER=plex_user \
-e POSTGRES_PASSWORD="${DB_PASSWORD}" \
postgres:16-alpine
db_ready=false
for i in $(seq 1 30); do
if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
db_ready=true
break
fi
sleep 2
done
if [ "${db_ready}" != "true" ]; then
echo "❌ Database did not become ready"
dump_failure_context
exit 1
fi
docker run -d \
--name "${BACKEND_CONTAINER}" \
--network "${NETWORK_NAME}" \
-p 18000:8000 \
-e DATABASE_URL="${DB_URL}" \
-e ENVIRONMENT=production \
"${EXPECTED_DIGEST_REF}"
backend_ready=false
for i in $(seq 1 40); do
backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)"
if [ "${backend_running}" != "true" ]; then
echo "❌ Backend container exited before becoming healthy"
dump_failure_context
exit 1
fi
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
if [ "${health_code}" = "200" ]; then
backend_ready=true
break
fi
sleep 2
done
if [ "${backend_ready}" != "true" ]; then
echo "❌ Backend did not become healthy"
cat /tmp/blackbox-health.json 2>/dev/null || true
dump_failure_context
exit 1
fi
root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)"
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
echo "❌ Root endpoint validation failed"
cat /tmp/blackbox-root.json 2>/dev/null || true
dump_failure_context
exit 1
fi
compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)"
if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
echo "❌ Compatibility endpoint validation failed"
cat /tmp/blackbox-compatibility.json 2>/dev/null || true
dump_failure_context
exit 1
fi
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then
echo "❌ Health endpoint validation failed"
cat /tmp/blackbox-health.json 2>/dev/null || true
dump_failure_context
exit 1
fi
echo "✅ Runtime black-box integration checks passed"
} 2>&1 | tee "${LOG_FILE}"
TEST_STATUS=${PIPESTATUS[0]}
if [ "${TEST_STATUS}" -ne 0 ]; then
echo "❌ Integration tests failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of integration test output ---"
echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of runtime black-box integration output ---"
tail -n 200 "${LOG_FILE}" || true
exit "${TEST_STATUS}"
fi

View File

@@ -138,6 +138,8 @@ jobs:
timeout-minutes: 60
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
steps:
- name: Identify runner
@@ -251,6 +253,44 @@ jobs:
--image deployable-frontend:"${HEAD_SHA}" \
--profile frontend
- name: Push deployable backend runtime image
id: deployable_backend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
docker push "${DEPLOYABLE_BACKEND_TAG_REF}"
docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null
DEPLOYABLE_BACKEND_DIGEST_REF="$({
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
| head -n 1
} || true)"
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve deployable backend digest reference"
exit 1
fi
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}"
- name: Build and push complete CICD image
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
@@ -347,6 +387,8 @@ jobs:
ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }}
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
HEAD_SHA: ${{ needs.build.outputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }}
REPO_FULL: ${{ github.repository }}
HEAD_REF: ${{ github.head_ref }}
REF_NAME: ${{ github.ref_name }}
@@ -426,6 +468,8 @@ jobs:
--head-sha "${HEAD_SHA}"
--source-workflow "CICD Main Build"
--trace-id "${TRACE_ID}"
--input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}"
--input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}"
)
for API_BASE in "${CANDIDATE_API_BASES[@]}"; do

View File

@@ -215,7 +215,7 @@ jobs:
cat > renovate-config.js << 'EOF'
module.exports = {
platform: 'gitea',
endpoint: 'https://dogar.darkhelm.org/api/v1',
endpoint: 'https://dogar.darkhelm.org',
gitAuthor: 'Renovate Bot <renovate@darkhelm.org>',
repositories: ['DarkHelm.org/plex-playlist'],
onboarding: false,
@@ -237,36 +237,25 @@ jobs:
RENOVATE_DRY_RUN: ${{ inputs.dry_run }}
RENOVATE_CONFIG_FILE: renovate-config.js
RENOVATE_PLATFORM: gitea
RENOVATE_ENDPOINT: https://dogar.darkhelm.org/api/v1
RENOVATE_ENDPOINT: https://dogar.darkhelm.org
RENOVATE_ALLOW_INSECURE_TLS: "true"
LOG_LEVEL: info
run: |
echo "=== Running Renovate Bot ==="
TARGET_REPO="DarkHelm.org/plex-playlist"
TARGET_ORG="${TARGET_REPO%%/*}"
select_token() {
if [ -n "${RENOVATE_TOKEN_SECRET:-}" ]; then
echo "${RENOVATE_TOKEN_SECRET}"
return 0
fi
if [ -n "${ACTIONS_TRIGGER_TOKEN:-}" ]; then
echo "${ACTIONS_TRIGGER_TOKEN}"
return 0
fi
if [ -n "${PACKAGE_ACCESS_TOKEN:-}" ]; then
echo "${PACKAGE_ACCESS_TOKEN}"
return 0
fi
return 1
}
if ! SELECTED_TOKEN="$(select_token)"; then
echo "❌ No token available for Renovate authentication"
echo "Configure one of: RENOVATE_TOKEN, ACTIONS_TRIGGER_TOKEN, PACKAGE_ACCESS_TOKEN"
exit 1
RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}"
if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then
API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}"
RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}"
else
API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1"
fi
export RENOVATE_TOKEN="${SELECTED_TOKEN}"
unset SELECTED_TOKEN
export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}"
echo "Renovate endpoint: ${RENOVATE_ENDPOINT}"
echo "Preflight API endpoint: ${API_ENDPOINT}"
CURL_INSECURE_FLAG=""
if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then
@@ -275,27 +264,71 @@ jobs:
export NODE_TLS_REJECT_UNAUTHORIZED=0
fi
# Validate token before starting Renovate to fail with actionable diagnostics.
AUTH_CHECK_STATUS=$(curl -sS -o /tmp/renovate-auth-check.json -w "%{http_code}" \
${CURL_INSECURE_FLAG} \
-H "Authorization: token ${RENOVATE_TOKEN}" \
"${RENOVATE_ENDPOINT}/user" || true)
select_token_with_repo_access() {
for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do
candidate_value="${!candidate_name:-}"
if [ -z "${candidate_value}" ]; then
continue
fi
if [ "${AUTH_CHECK_STATUS}" != "200" ]; then
echo "❌ Renovate token authentication preflight failed (HTTP ${AUTH_CHECK_STATUS})"
echo "Expected a personal access token with repo/issue read-write permissions."
if [ -s /tmp/renovate-auth-check.json ]; then
echo "Response body:"
cat /tmp/renovate-auth-check.json || true
USER_STATUS=$(curl -sS -o /tmp/renovate-auth-check-user.json -w "%{http_code}" \
${CURL_INSECURE_FLAG} \
-H "Authorization: token ${candidate_value}" \
"${API_ENDPOINT}/user" || true)
REPO_STATUS=$(curl -sS -o /tmp/renovate-auth-check-repo.json -w "%{http_code}" \
${CURL_INSECURE_FLAG} \
-H "Authorization: token ${candidate_value}" \
"${API_ENDPOINT}/repos/${TARGET_REPO}" || true)
ORG_STATUS=$(curl -sS -o /tmp/renovate-auth-check-org.json -w "%{http_code}" \
${CURL_INSECURE_FLAG} \
-H "Authorization: token ${candidate_value}" \
"${API_ENDPOINT}/orgs/${TARGET_ORG}" || true)
if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then
echo "${candidate_name}:${candidate_value}"
return 0
fi
echo "⚠ Token candidate ${candidate_name} rejected (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})"
done
return 1
}
if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then
echo "❌ No token available for Renovate authentication with repository access"
echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes."
echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}"
if [ -s /tmp/renovate-auth-check-user.json ]; then
echo "Last user endpoint response body:"
cat /tmp/renovate-auth-check-user.json || true
fi
if [ -s /tmp/renovate-auth-check-repo.json ]; then
echo "Last repo endpoint response body:"
cat /tmp/renovate-auth-check-repo.json || true
fi
if [ -s /tmp/renovate-auth-check-org.json ]; then
echo "Last org endpoint response body:"
cat /tmp/renovate-auth-check-org.json || true
fi
exit 1
fi
echo "✓ Renovate auth preflight passed"
SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}"
SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}"
export RENOVATE_TOKEN="${SELECTED_TOKEN}"
unset SELECTED_TOKEN RESULT_TOKEN
echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}"
# Run Renovate with configuration
if [ "${RENOVATE_DRY_RUN}" = "true" ]; then
export RENOVATE_DRY_RUN="full"
echo "🔍 Running in DRY-RUN mode (no changes will be made)"
else
unset RENOVATE_DRY_RUN
fi
renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist

View File

@@ -34,4 +34,4 @@ COPY backend/ .
EXPOSE 8000
# Default command - can be overridden in compose for development
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"]

View File

@@ -55,6 +55,7 @@ RUN --mount=type=secret,id=ssh_private_key \
# Extract only dependency files for caching optimization
mkdir -p /workspace/backend /workspace/frontend && \
cp /tmp/repo/backend/pyproject.toml /workspace/backend/ 2>/dev/null || echo "No backend pyproject.toml" && \
cp /tmp/repo/backend/uv.lock /workspace/backend/ 2>/dev/null || echo "No backend uv.lock" && \
cp /tmp/repo/frontend/package.json /workspace/frontend/ 2>/dev/null || echo "No frontend package.json" && \
cp /tmp/repo/frontend/yarn.lock /workspace/frontend/ 2>/dev/null || echo "No frontend yarn.lock" && \
cp /tmp/repo/frontend/.yarnrc.yml /workspace/frontend/ 2>/dev/null || echo "No frontend .yarnrc.yml" && \
@@ -90,7 +91,13 @@ RUN echo "=== Installing Backend Dependencies (Phase 1: Optimized Caching) ==="
echo "# Temporary README for dependency caching phase" > ../README.md && \
echo "# Minimal __init__.py for build" > src/backend/__init__.py && \
# Install all dependencies including dev dependencies
uv sync --dev && \
if [ -f "uv.lock" ]; then \
echo "Using locked backend dependencies (uv.lock)" && \
uv sync --dev --frozen; \
else \
echo "uv.lock not found, resolving dependencies from pyproject.toml" && \
uv sync --dev; \
fi && \
echo "✓ Backend dependencies installed and cached"; \
else \
echo "No pyproject.toml found, skipping dependency installation"; \

View File

@@ -22,7 +22,9 @@ services:
- "8001:8000"
command:
- "uvicorn"
- "main:app"
- "backend.main:app"
- "--app-dir"
- "/app/src"
- "--host"
- "0.0.0.0"
- "--port"

View File

@@ -33,6 +33,8 @@
- **E2E Tests**: Simplified Docker approach matching other successful test patterns
- **Playwright**: Chromium-only CI strategy (95%+ browser market coverage)
- **Registry Operations**: Consistent approach across all test phases
- **Backend Runtime Integration**: Black-box checks run against started deployable
backend containers with commit-tag and digest pinning
## 🛠️ **Critical Issues Resolved**
@@ -82,6 +84,8 @@ Frontend Environment (Yarn PnP)
↓ (state regeneration)
Test Execution (all phases)
↓ (consistent Docker approach)
Runtime Black-Box Integration
↓ (deployable backend tag+digest verification)
E2E Testing (Playwright)
↓ (Chromium + network resilience)
✅ SUCCESS

View File

@@ -31,7 +31,7 @@ Excluded:
- Container build source: `Dockerfile.backend`.
- Runtime base image: `python:3.14-slim`.
- Runtime process: `uvicorn main:app --host 0.0.0.0 --port 8000`.
- Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`.
- Exposed runtime port: `8000`.
### Required Runtime Dependencies
@@ -68,7 +68,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth.
### Backend Runtime Checklist
- [ ] Runtime image built from `Dockerfile.backend`.
- [ ] Runtime process is uvicorn serving `main:app` on `0.0.0.0:8000`.
- [ ] Runtime process is uvicorn serving `backend.main:app` with app dir `/app/src` on `0.0.0.0:8000`.
- [ ] `DATABASE_URL` is set in deployment runtime.
- [ ] `GET /health` behavior matches contract.
- [ ] Startup fails on runtime policy mismatch.
@@ -157,6 +157,12 @@ Current enforcement implemented in CI:
- Workflow: `.gitea/workflows/docker-build-main.yaml`
- Checks include binary presence and profile-specific package metadata probes
to detect CI/development tooling leakage.
- Runtime black-box integration checks against deployed backend container:
- Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job)
- Inputs: deployable backend commit tag reference and immutable digest
reference from main build dispatch.
- Assertions: digest/tag consistency and live endpoint behavior for `/`,
`/compatibility`, and `/health` on started runtime containers.
Future follow-up automation under epic #66 may expand this with:

View File

@@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />

View File

@@ -4,6 +4,7 @@
import os
import subprocess
import sys
import tempfile
from datetime import datetime
@@ -17,6 +18,208 @@ HOSTS = {
"pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
}
DEFAULT_MEM_FRACTION = 0.75
DEFAULT_JOB_MEM_FRACTION = 0.9
DEFAULT_HOST_MEM_SHARE = 0.25
DEFAULT_HOST_CPU_SHARE = 0.25
DEFAULT_CPU_FRACTION = 1.0
DEFAULT_JOB_CPU_FRACTION = 0.9
DEFAULT_RUNNER_MEM_SWAP_FACTOR = 1.0
DEFAULT_JOB_MEM_SWAP_FACTOR = 1.0
DEFAULT_RUNNER_PIDS_LIMIT = 1024
DEFAULT_JOB_PIDS_LIMIT = 512
DEFAULT_JOB_NOFILE_SOFT = 2048
DEFAULT_JOB_NOFILE_HARD = 4096
DEFAULT_RUNNER_MAX_PARALLEL_JOBS = 1
MIN_RUNNER_MEM_MIB = 256
MIN_JOB_MEM_MIB = 128
def parse_mem_fraction(argv):
fraction = DEFAULT_MEM_FRACTION
for arg in argv[1:]:
if arg.startswith("--mem-fraction="):
_, value = arg.split("=", 1)
try:
fraction = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --mem-fraction value: {value}") from exc
if fraction <= 0 or fraction > 1:
raise SystemExit("--mem-fraction must be > 0 and <= 1")
return fraction
def parse_job_mem_fraction(argv):
fraction = DEFAULT_JOB_MEM_FRACTION
for arg in argv[1:]:
if arg.startswith("--job-mem-fraction="):
_, value = arg.split("=", 1)
try:
fraction = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --job-mem-fraction value: {value}") from exc
if fraction <= 0 or fraction > 1:
raise SystemExit("--job-mem-fraction must be > 0 and <= 1")
return fraction
def parse_host_mem_share(argv):
share = DEFAULT_HOST_MEM_SHARE
for arg in argv[1:]:
if arg.startswith("--host-mem-share="):
_, value = arg.split("=", 1)
try:
share = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --host-mem-share value: {value}") from exc
if share <= 0 or share > 1:
raise SystemExit("--host-mem-share must be > 0 and <= 1")
return share
def parse_host_cpu_share(argv):
share = DEFAULT_HOST_CPU_SHARE
for arg in argv[1:]:
if arg.startswith("--host-cpu-share="):
_, value = arg.split("=", 1)
try:
share = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --host-cpu-share value: {value}") from exc
if share <= 0 or share > 1:
raise SystemExit("--host-cpu-share must be > 0 and <= 1")
return share
def parse_cpu_fraction(argv):
fraction = DEFAULT_CPU_FRACTION
for arg in argv[1:]:
if arg.startswith("--cpu-fraction="):
_, value = arg.split("=", 1)
try:
fraction = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --cpu-fraction value: {value}") from exc
if fraction <= 0 or fraction > 1:
raise SystemExit("--cpu-fraction must be > 0 and <= 1")
return fraction
def parse_job_cpu_fraction(argv):
fraction = DEFAULT_JOB_CPU_FRACTION
for arg in argv[1:]:
if arg.startswith("--job-cpu-fraction="):
_, value = arg.split("=", 1)
try:
fraction = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --job-cpu-fraction value: {value}") from exc
if fraction <= 0 or fraction > 1:
raise SystemExit("--job-cpu-fraction must be > 0 and <= 1")
return fraction
def parse_runner_mem_swap_factor(argv):
factor = DEFAULT_RUNNER_MEM_SWAP_FACTOR
for arg in argv[1:]:
if arg.startswith("--runner-memory-swap-factor="):
_, value = arg.split("=", 1)
try:
factor = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --runner-memory-swap-factor value: {value}") from exc
if factor < 1:
raise SystemExit("--runner-memory-swap-factor must be >= 1")
return factor
def parse_job_mem_swap_factor(argv):
factor = DEFAULT_JOB_MEM_SWAP_FACTOR
for arg in argv[1:]:
if arg.startswith("--job-memory-swap-factor="):
_, value = arg.split("=", 1)
try:
factor = float(value)
except ValueError as exc:
raise SystemExit(f"Invalid --job-memory-swap-factor value: {value}") from exc
if factor < 1:
raise SystemExit("--job-memory-swap-factor must be >= 1")
return factor
def parse_runner_pids_limit(argv):
value = DEFAULT_RUNNER_PIDS_LIMIT
for arg in argv[1:]:
if arg.startswith("--runner-pids-limit="):
_, raw = arg.split("=", 1)
try:
value = int(raw)
except ValueError as exc:
raise SystemExit(f"Invalid --runner-pids-limit value: {raw}") from exc
if value < 64:
raise SystemExit("--runner-pids-limit must be >= 64")
return value
def parse_job_pids_limit(argv):
value = DEFAULT_JOB_PIDS_LIMIT
for arg in argv[1:]:
if arg.startswith("--job-pids-limit="):
_, raw = arg.split("=", 1)
try:
value = int(raw)
except ValueError as exc:
raise SystemExit(f"Invalid --job-pids-limit value: {raw}") from exc
if value < 32:
raise SystemExit("--job-pids-limit must be >= 32")
return value
def parse_job_nofile_soft(argv):
value = DEFAULT_JOB_NOFILE_SOFT
for arg in argv[1:]:
if arg.startswith("--job-ulimit-nofile-soft="):
_, raw = arg.split("=", 1)
try:
value = int(raw)
except ValueError as exc:
raise SystemExit(f"Invalid --job-ulimit-nofile-soft value: {raw}") from exc
if value < 256:
raise SystemExit("--job-ulimit-nofile-soft must be >= 256")
return value
def parse_job_nofile_hard(argv):
value = DEFAULT_JOB_NOFILE_HARD
for arg in argv[1:]:
if arg.startswith("--job-ulimit-nofile-hard="):
_, raw = arg.split("=", 1)
try:
value = int(raw)
except ValueError as exc:
raise SystemExit(f"Invalid --job-ulimit-nofile-hard value: {raw}") from exc
if value < 256:
raise SystemExit("--job-ulimit-nofile-hard must be >= 256")
return value
def parse_runner_max_parallel_jobs(argv):
value = DEFAULT_RUNNER_MAX_PARALLEL_JOBS
for arg in argv[1:]:
if arg.startswith("--runner-max-parallel-jobs="):
_, raw = arg.split("=", 1)
try:
value = int(raw)
except ValueError as exc:
raise SystemExit(f"Invalid --runner-max-parallel-jobs value: {raw}") from exc
if value < 1:
raise SystemExit("--runner-max-parallel-jobs must be >= 1")
return value
def parse_dry_run(argv):
return "--dry-run" in argv[1:]
def ssh(host, cmd):
proc = subprocess.run(
[
@@ -55,7 +258,238 @@ def scp(local_path, host, remote_path):
return proc.returncode, proc.stdout + proc.stderr
def build_compose(compose_text):
def service_block_bounds(lines, start_index):
service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" "))
block_start = start_index + 1
block_end = len(lines)
j = block_start
while j < len(lines):
cur = lines[j]
cur_stripped = cur.strip()
if cur_stripped:
cur_indent = len(cur) - len(cur.lstrip(" "))
if cur_indent <= service_indent:
block_end = j
break
j += 1
return service_indent, block_start, block_end
def upsert_service_key(compose_text, service_names, key, value):
lines = compose_text.splitlines()
i = 0
while i < len(lines):
stripped = lines[i].strip()
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
i += 1
continue
service_indent, block_start, block_end = service_block_bounds(lines, i)
field_indent = " " * (service_indent + 2)
key_prefix = f"{key}:"
found_idx = None
for k in range(block_start, block_end):
if lines[k].strip().startswith(key_prefix):
found_idx = k
break
new_line = f"{field_indent}{key}: {value}"
if found_idx is not None:
lines[found_idx] = new_line
i = block_end
continue
lines.insert(block_start, new_line)
i = block_end + 1
return "\n".join(lines) + "\n"
def remove_service_key(compose_text, service_names, key):
lines = compose_text.splitlines()
i = 0
while i < len(lines):
stripped = lines[i].strip()
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
i += 1
continue
_, block_start, block_end = service_block_bounds(lines, i)
key_prefix = f"{key}:"
filtered = []
for idx in range(block_start, block_end):
if not lines[idx].strip().startswith(key_prefix):
filtered.append(lines[idx])
lines = lines[:block_start] + filtered + lines[block_end:]
i = block_start + len(filtered)
return "\n".join(lines) + "\n"
def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value, pids_value):
lines = compose_text.splitlines()
i = 0
while i < len(lines):
stripped = lines[i].strip()
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
i += 1
continue
service_indent, block_start, service_block_end = service_block_bounds(lines, i)
deploy_indent = " " * (service_indent + 2)
resources_indent = " " * (service_indent + 4)
limits_indent = " " * (service_indent + 6)
value_indent = " " * (service_indent + 8)
deploy_prefix = f"{deploy_indent}deploy:"
# Remove an existing deploy block under this service so we can rewrite it canonically.
deploy_idx = None
deploy_end = None
for idx in range(block_start, service_block_end):
if lines[idx].startswith(deploy_prefix):
deploy_idx = idx
j = idx + 1
while j < service_block_end:
cur = lines[j]
cur_stripped = cur.strip()
if cur_stripped:
cur_indent = len(cur) - len(cur.lstrip(" "))
if cur_indent <= service_indent + 2:
break
j += 1
deploy_end = j
break
if deploy_idx is not None and deploy_end is not None:
lines = lines[:deploy_idx] + lines[deploy_end:]
service_block_end -= deploy_end - deploy_idx
insert_lines = [
f"{deploy_indent}deploy:",
f"{resources_indent}resources:",
f"{limits_indent}limits:",
f"{value_indent}memory: {memory_value}",
f"{value_indent}cpus: '{cpu_value}'",
f"{value_indent}pids: {pids_value}",
]
lines = lines[:block_start] + insert_lines + lines[block_start:]
i = service_block_end + len(insert_lines)
return "\n".join(lines) + "\n"
def upsert_compose_env_var(compose_text, var_name, var_value, anchor_line):
lines = compose_text.splitlines()
new_line = f" - {var_name}={var_value}"
replaced = False
for idx, line in enumerate(lines):
stripped = line.strip()
if stripped.startswith(f"- {var_name}="):
lines[idx] = new_line
replaced = True
if not replaced:
for idx, line in enumerate(lines):
if line.strip() == anchor_line.strip():
lines.insert(idx + 1, new_line)
return "\n".join(lines) + "\n"
def parse_mem_value_to_mib(raw_value):
value = raw_value.strip().lower()
if not value:
return None
# Compose mem_limit values are typically 4096m / 4g; inspect values are bytes.
units = {
"k": 1.0 / 1024.0,
"m": 1.0,
"g": 1024.0,
}
for suffix, factor in units.items():
if value.endswith(suffix):
num = value[: -len(suffix)]
try:
return int(float(num) * factor)
except ValueError:
return None
if value.isdigit():
as_int = int(value)
# Docker inspect HostConfig.Memory reports bytes.
if as_int > 0:
return int(as_int / (1024 * 1024))
return None
def find_service_key_value(compose_text, service_names, key):
lines = compose_text.splitlines()
i = 0
while i < len(lines):
stripped = lines[i].strip()
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
i += 1
continue
service_indent = len(lines[i]) - len(lines[i].lstrip(" "))
j = i + 1
while j < len(lines):
cur = lines[j]
cur_stripped = cur.strip()
if cur_stripped:
cur_indent = len(cur) - len(cur.lstrip(" "))
if cur_indent <= service_indent:
break
if cur_stripped.startswith(f"{key}:"):
return cur_stripped.split(":", 1)[1].strip()
j += 1
i = j
return None
def find_service_memory_value(compose_text, service_names):
lines = compose_text.splitlines()
i = 0
while i < len(lines):
stripped = lines[i].strip()
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
i += 1
continue
_, block_start, block_end = service_block_bounds(lines, i)
for idx in range(block_start, block_end):
cur = lines[idx].strip()
if cur.startswith("memory:"):
return cur.split(":", 1)[1].strip()
i = block_end
return None
def build_compose(
compose_text,
runner_mem_limit,
runner_memswap_limit,
runner_cpu_limit,
runner_pids_limit,
runner_max_parallel_jobs,
job_mem_limit,
job_memswap_limit,
job_cpu_limit,
job_pids_limit,
job_nofile_soft,
job_nofile_hard,
):
updated = compose_text
volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro"
@@ -65,12 +499,121 @@ def build_compose(compose_text):
force_pull_line = " - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}"
add_cfg_env = " - CONFIG_FILE=/config.yaml"
if add_cfg_env not in updated and force_pull_line in updated:
if force_pull_line in updated:
updated = upsert_compose_env_var(updated, "CONFIG_FILE", "/config.yaml", force_pull_line)
updated = upsert_compose_env_var(
updated,
"GITEA_RUNNER_MAX_PARALLEL_JOBS",
str(runner_max_parallel_jobs),
force_pull_line,
)
updated = upsert_compose_env_var(
updated,
"GITEA_RUNNER_JOB_CONTAINER_OPTIONS",
(
f"--memory={job_mem_limit} --memory-swap={job_memswap_limit} "
f"--cpus={job_cpu_limit} --pids-limit={job_pids_limit} "
f"--ulimit=nofile={job_nofile_soft}:{job_nofile_hard}"
),
force_pull_line,
)
elif add_cfg_env not in updated:
updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env)
runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"}
updated = remove_service_key(updated, runner_services, "mem_limit")
updated = remove_service_key(updated, runner_services, "pids_limit")
updated = remove_service_key(updated, runner_services, "pids")
updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit)
updated = upsert_service_deploy_limits(
updated,
runner_services,
runner_mem_limit,
runner_cpu_limit,
str(runner_pids_limit),
)
return updated
def host_total_mem_mib(host):
rc, mem_kib = ssh(host, "awk '/MemTotal:/ {print $2}' /proc/meminfo")
if rc != 0 or not mem_kib.strip().isdigit():
raise RuntimeError(f"Could not read MemTotal on {host}")
return int(int(mem_kib.strip()) / 1024.0)
def host_total_cpus(host):
rc, cpu_count = ssh(host, "nproc")
if rc != 0:
raise RuntimeError(f"Could not read CPU count on {host}")
value = cpu_count.strip()
try:
cpus = float(value)
except ValueError as exc:
raise RuntimeError(f"Invalid CPU count on {host}: {value}") from exc
if cpus <= 0:
raise RuntimeError(f"Non-positive CPU count on {host}: {value}")
return cpus
def host_mem_limit(host, host_mem_share, fraction):
total_mib = host_total_mem_mib(host)
host_share_mib = int(total_mib * host_mem_share)
target_mib = int(host_share_mib * fraction)
if target_mib < MIN_RUNNER_MEM_MIB:
target_mib = MIN_RUNNER_MEM_MIB
return (
f"{target_mib}m",
target_mib,
f"host-total({total_mib}MiB) * host-share({host_mem_share:.2f}) -> {host_share_mib}MiB * mem-fraction({fraction:.2f})",
)
def host_cpu_limit(host, host_cpu_share, fraction):
total_cpus = host_total_cpus(host)
host_share_cpus = total_cpus * host_cpu_share
target_cpus = host_share_cpus * fraction
if target_cpus <= 0.01:
target_cpus = 0.01
return (
f"{target_cpus:.2f}",
target_cpus,
f"host-total({total_cpus:.0f}cpu) * host-share({host_cpu_share:.2f}) -> {host_share_cpus:.2f}cpu * cpu-fraction({fraction:.2f})",
)
mem_fraction = parse_mem_fraction(sys.argv)
job_mem_fraction = parse_job_mem_fraction(sys.argv)
host_mem_share = parse_host_mem_share(sys.argv)
host_cpu_share = parse_host_cpu_share(sys.argv)
cpu_fraction = parse_cpu_fraction(sys.argv)
job_cpu_fraction = parse_job_cpu_fraction(sys.argv)
runner_mem_swap_factor = parse_runner_mem_swap_factor(sys.argv)
job_mem_swap_factor = parse_job_mem_swap_factor(sys.argv)
runner_pids_limit = parse_runner_pids_limit(sys.argv)
job_pids_limit = parse_job_pids_limit(sys.argv)
job_nofile_soft = parse_job_nofile_soft(sys.argv)
job_nofile_hard = parse_job_nofile_hard(sys.argv)
runner_max_parallel_jobs = parse_runner_max_parallel_jobs(sys.argv)
dry_run = parse_dry_run(sys.argv)
if job_nofile_hard < job_nofile_soft:
raise SystemExit("--job-ulimit-nofile-hard must be >= --job-ulimit-nofile-soft")
print(f"Host mem share: {host_mem_share:.2f}")
print(f"Runner mem fraction: {mem_fraction:.2f}")
print(f"Job mem fraction: {job_mem_fraction:.2f}")
print(f"Host cpu share: {host_cpu_share:.2f}")
print(f"Runner cpu fraction: {cpu_fraction:.2f}")
print(f"Job cpu fraction: {job_cpu_fraction:.2f}")
print(f"Runner memory swap factor: {runner_mem_swap_factor:.2f}")
print(f"Job memory swap factor: {job_mem_swap_factor:.2f}")
print(f"Runner pids limit: {runner_pids_limit}")
print(f"Job pids limit: {job_pids_limit}")
print(f"Job nofile soft/hard: {job_nofile_soft}/{job_nofile_hard}")
print(f"Runner max parallel jobs: {runner_max_parallel_jobs}")
print(f"Dry run: {dry_run}")
print("Fetching canonical runner config template")
rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}")
if rc != 0 or not template.strip():
@@ -92,7 +635,73 @@ with tempfile.TemporaryDirectory() as tmpdir:
print("Could not read compose.yml")
continue
new_compose = build_compose(compose)
try:
runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit(
host,
host_mem_share,
mem_fraction,
)
runner_cpu_limit, runner_cpu_value, cpu_source = host_cpu_limit(
host,
host_cpu_share,
cpu_fraction,
)
except RuntimeError as exc:
print(str(exc))
continue
print(f"runner_mem_baseline={baseline_source}")
print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)")
runner_memswap_mib = int(runner_mem_mib * runner_mem_swap_factor)
if runner_memswap_mib < runner_mem_mib:
runner_memswap_mib = runner_mem_mib
runner_memswap_limit = f"{runner_memswap_mib}m"
print(f"runner_memswap_limit={runner_memswap_limit} ({runner_memswap_mib} MiB)")
print(f"runner_cpu_baseline={cpu_source}")
print(f"runner_cpu_limit={runner_cpu_limit} cpu")
job_mem_mib = int(runner_mem_mib * job_mem_fraction)
if job_mem_mib < MIN_JOB_MEM_MIB:
job_mem_mib = MIN_JOB_MEM_MIB
if job_mem_mib >= runner_mem_mib and runner_mem_mib > 1:
job_mem_mib = runner_mem_mib - 1
job_mem_limit = f"{job_mem_mib}m"
print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)")
job_memswap_mib = int(job_mem_mib * job_mem_swap_factor)
if job_memswap_mib < job_mem_mib:
job_memswap_mib = job_mem_mib
job_memswap_limit = f"{job_memswap_mib}m"
print(f"job_memswap_limit={job_memswap_limit} ({job_memswap_mib} MiB)")
job_cpu_value = runner_cpu_value * job_cpu_fraction
if job_cpu_value <= 0.01:
job_cpu_value = 0.01
if job_cpu_value >= runner_cpu_value and runner_cpu_value > 0.02:
job_cpu_value = runner_cpu_value - 0.01
job_cpu_limit = f"{job_cpu_value:.2f}"
print(f"job_cpu_limit={job_cpu_limit} cpu")
print(f"job_pids_limit={job_pids_limit}")
print(f"job_nofile_soft/hard={job_nofile_soft}/{job_nofile_hard}")
new_compose = build_compose(
compose,
runner_mem_limit,
runner_memswap_limit,
runner_cpu_limit,
runner_pids_limit,
runner_max_parallel_jobs,
job_mem_limit,
job_memswap_limit,
job_cpu_limit,
job_pids_limit,
job_nofile_soft,
job_nofile_hard,
)
if dry_run:
print("dry-run=true; skipping upload and restart")
continue
rc, _ = ssh(host, f"mkdir -p {root}/backups")
rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}")
@@ -120,8 +729,18 @@ with tempfile.TemporaryDirectory() as tmpdir:
print(out.strip())
rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml")
print(out.strip())
rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml")
print(out.strip())
rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_MAX_PARALLEL_JOBS=' {root}/compose.yml")
print(out.strip())
rc, out = ssh(host, f"grep -nE 'memory:|cpus:|pids:|memswap_limit:|pids_limit:' {root}/compose.yml")
print(out.strip())
rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1")
if rc != 0:
print(out.strip())
print("Failed to apply compose changes on host")
continue
print(out.strip())
rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")