Files
plex-playlist/.gitea/workflows/cicd-tests.yaml
copilotcoder 6da60e6beb
All checks were successful
CICD Start / Sanity and Base Decision (push) Successful in 21s
CICD Start / Sanity and Base Decision (pull_request) Successful in 40s
fix(ci): correct backend ASGI path for runtime black-box lane
2026-07-03 21:58:36 -04:00

539 lines
22 KiB
YAML

name: CICD Tests
on:
workflow_dispatch:
inputs:
head_sha:
description: Commit SHA to process
required: false
deployable_backend_tag_ref:
description: Deployable backend image tag reference
required: false
deployable_backend_digest_ref:
description: Deployable backend image digest reference
required: false
source_workflow:
description: Upstream workflow name
required: false
trace_id:
description: Correlation id propagated across CICD dispatch chain
required: false
env:
GITEA_REGISTRY: kankali.darkhelm.lan:3001
GITEA_REGISTRY_IP: 10.18.75.2
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
concurrency:
group: tests-${{ github.sha }}
cancel-in-progress: true
jobs:
setup:
name: Setup Tests Context
# Use the same stable runner label as the test jobs.
runs-on: ubuntu-act
timeout-minutes: 8
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }}
deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }}
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- name: Audit trigger context
env:
EVENT_NAME: ${{ github.event_name }}
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }}
HEAD_SHA_FALLBACK: ${{ github.sha }}
REF: ${{ github.ref }}
REF_NAME: ${{ github.ref_name }}
HEAD_REF: ${{ github.head_ref }}
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
run: |
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
TRACE_ID="${TRACE_ID_INPUT:-cicd-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}"
echo "=== Dispatch Audit: CICD Tests ==="
echo "event_name=${EVENT_NAME}"
echo "source_workflow=${SOURCE_WORKFLOW}"
echo "head_sha_input=${HEAD_SHA_INPUT}"
echo "head_sha=${RESOLVED_HEAD_SHA}"
echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}"
echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}"
echo "ref=${REF}"
echo "ref_name=${REF_NAME}"
echo "head_ref=${HEAD_REF}"
echo "trace_id=${TRACE_ID}"
- name: Resolve head SHA
id: meta
env:
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }}
HEAD_SHA_FALLBACK: ${{ github.sha }}
run: |
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT"
- &failure_diagnostics_step
name: Failure diagnostics
if: failure()
run: |
echo "=== Failure Diagnostics ==="
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
echo "runner_name=${RUNNER_NAME:-unknown}"
echo "runner_hostname=${HOSTNAME:-unknown}"
uname -a || true
cat /etc/os-release 2>/dev/null || true
df -h || true
free -h || true
ps aux --sort=-%mem | head -n 30 || true
if command -v docker >/dev/null 2>&1; then
echo "=== Docker Diagnostics ==="
docker version || true
docker info || true
docker ps -a || true
docker images --digests | head -n 50 || true
else
echo "docker not available on this runner"
fi
echo "=== Kernel Tail ==="
dmesg | tail -n 120 || true
backend-tests:
name: Backend Tests
# Use ubuntu-act runner pool for consistent availability.
runs-on: ubuntu-act
timeout-minutes: 25
needs: setup
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- &configure_registry_host_step
name: Configure registry host resolution
run: |
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
- &ensure_cicd_image_step
name: Ensure CICD image is available
env:
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
run: |
IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
echo "Using cached CICD image: ${IMAGE}"
else
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
pulled=false
for i in 1 2 3; do
echo "Pull attempt ${i}/3 for ${IMAGE}"
if docker pull "${IMAGE}"; then
pulled=true
break
fi
if [ "${i}" -lt 3 ]; then
sleep_seconds=$((5 * i))
echo "Pull failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
done
if [ "${pulled}" != "true" ]; then
echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
exit 1
fi
fi
- name: Run backend tests with coverage
env:
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
run: |
set -o pipefail
LOG_FILE="$(mktemp)"
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
cd /workspace/backend &&
source .venv/bin/activate &&
uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95
" 2>&1 | tee "${LOG_FILE}"
TEST_STATUS=${PIPESTATUS[0]}
if [ "${TEST_STATUS}" -ne 0 ]; then
echo "❌ Backend tests failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of backend test output ---"
tail -n 200 "${LOG_FILE}" || true
exit "${TEST_STATUS}"
fi
- *failure_diagnostics_step
frontend-tests:
name: Frontend Tests
# Use ubuntu-act runner pool for consistent availability.
runs-on: ubuntu-act
timeout-minutes: 25
needs: setup
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- *configure_registry_host_step
- *ensure_cicd_image_step
- name: Run frontend tests with coverage
env:
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
run: |
set -o pipefail
LOG_FILE="$(mktemp)"
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
cd /workspace/frontend &&
yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85
" 2>&1 | tee "${LOG_FILE}"
TEST_STATUS=${PIPESTATUS[0]}
if [ "${TEST_STATUS}" -ne 0 ]; then
echo "❌ Frontend tests failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of frontend test output ---"
tail -n 200 "${LOG_FILE}" || true
exit "${TEST_STATUS}"
fi
- *failure_diagnostics_step
xdoctest:
name: Backend Doctests
# Use ubuntu-act runner pool for consistent availability.
runs-on: ubuntu-act
timeout-minutes: 15
needs: setup
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- *configure_registry_host_step
- *ensure_cicd_image_step
- name: Run backend doctests
env:
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
run: |
set -o pipefail
LOG_FILE="$(mktemp)"
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
cd /workspace/backend &&
source .venv/bin/activate &&
uv run xdoctest src/ --quiet
" 2>&1 | tee "${LOG_FILE}"
TEST_STATUS=${PIPESTATUS[0]}
if [ "${TEST_STATUS}" -ne 0 ]; then
echo "❌ Backend doctests failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of doctest output ---"
tail -n 200 "${LOG_FILE}" || true
exit "${TEST_STATUS}"
fi
- *failure_diagnostics_step
integration-tests:
name: Runtime Black-Box Integration Tests
# Pin integration tests to high-memory worker to reduce setup-stage runner churn.
runs-on: ubuntu-act-8gb
timeout-minutes: 20
needs: [setup, backend-tests]
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- *configure_registry_host_step
- name: Run runtime black-box integration checks
env:
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }}
run: |
set -euo pipefail
set -o pipefail
RUN_ID="${GITHUB_RUN_ID:-local}"
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
LOG_FILE="$(mktemp)"
NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}"
DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}"
BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}"
DB_PASSWORD="plex_password"
DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist"
dump_failure_context() {
echo "=== Runtime Integration Failure Context ==="
docker ps -a || true
echo "--- Backend logs (tail 200) ---"
docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true
echo "--- Database logs (tail 200) ---"
docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true
echo "--- Backend inspect (status/image) ---"
docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true
echo "--- Database inspect (status/image) ---"
docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true
}
cleanup() {
docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true
docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true
docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true
}
trap cleanup EXIT
{
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
echo "❌ Missing deployable backend image references from dispatch inputs"
exit 1
fi
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
docker pull "${DEPLOYABLE_BACKEND_TAG_REF}"
docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}"
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
ACTUAL_TAG_DIGEST_REF="$({
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
| head -n 1
} || true)"
if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then
echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
exit 1
fi
if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then
echo "❌ Tag and digest mismatch"
echo "expected=${EXPECTED_DIGEST_REF}"
echo "actual=${ACTUAL_TAG_DIGEST_REF}"
exit 1
fi
echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}"
echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}"
docker network create "${NETWORK_NAME}"
docker run -d \
--name "${DB_CONTAINER}" \
--network "${NETWORK_NAME}" \
-e POSTGRES_DB=plex_playlist \
-e POSTGRES_USER=plex_user \
-e POSTGRES_PASSWORD="${DB_PASSWORD}" \
postgres:16-alpine
db_ready=false
for i in $(seq 1 30); do
if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
db_ready=true
break
fi
sleep 2
done
if [ "${db_ready}" != "true" ]; then
echo "❌ Database did not become ready"
dump_failure_context
exit 1
fi
docker run -d \
--name "${BACKEND_CONTAINER}" \
--network "${NETWORK_NAME}" \
-p 18000:8000 \
-e DATABASE_URL="${DB_URL}" \
-e ENVIRONMENT=production \
"${EXPECTED_DIGEST_REF}"
backend_ready=false
for i in $(seq 1 40); do
backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)"
if [ "${backend_running}" != "true" ]; then
echo "❌ Backend container exited before becoming healthy"
dump_failure_context
exit 1
fi
health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)"
if [ "${health_code}" = "200" ]; then
backend_ready=true
break
fi
sleep 2
done
if [ "${backend_ready}" != "true" ]; then
echo "❌ Backend did not become healthy"
cat /tmp/blackbox-health.json 2>/dev/null || true
dump_failure_context
exit 1
fi
root_code="$(curl -sS -o /tmp/blackbox-root.json -w "%{http_code}" "http://127.0.0.1:18000/" || true)"
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
echo "❌ Root endpoint validation failed"
cat /tmp/blackbox-root.json 2>/dev/null || true
dump_failure_context
exit 1
fi
compatibility_code="$(curl -sS -o /tmp/blackbox-compatibility.json -w "%{http_code}" "http://127.0.0.1:18000/compatibility" || true)"
if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
echo "❌ Compatibility endpoint validation failed"
cat /tmp/blackbox-compatibility.json 2>/dev/null || true
dump_failure_context
exit 1
fi
health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)"
if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then
echo "❌ Health endpoint validation failed"
cat /tmp/blackbox-health.json 2>/dev/null || true
dump_failure_context
exit 1
fi
echo "✅ Runtime black-box integration checks passed"
} 2>&1 | tee "${LOG_FILE}"
TEST_STATUS=${PIPESTATUS[0]}
if [ "${TEST_STATUS}" -ne 0 ]; then
echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of runtime black-box integration output ---"
tail -n 200 "${LOG_FILE}" || true
exit "${TEST_STATUS}"
fi
- *failure_diagnostics_step
e2e-tests:
name: End-to-End Tests
# Use ubuntu-act runner pool for consistent availability.
runs-on: ubuntu-act
timeout-minutes: 30
needs: [setup, frontend-tests]
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-}"
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
echo "runner_hostname_env=${HOSTNAME:-unknown}"
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
echo "runner_os=${RUNNER_OS:-unknown}"
echo "runner_arch=${RUNNER_ARCH:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- *configure_registry_host_step
- *ensure_cicd_image_step
- name: Run E2E tests
env:
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
run: |
set -o pipefail
LOG_FILE="$(mktemp)"
docker run --rm -e CI=true "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
cd /workspace/frontend &&
if [ -d 'tests/e2e' ] || grep -q 'playwright' package.json; then
yarn playwright --version &&
PW_BROWSER_PATH="\${PLAYWRIGHT_BROWSERS_PATH:-/root/.cache/ms-playwright}" &&
if find "\${PW_BROWSER_PATH}" -maxdepth 1 -type d -name 'chromium-*' | grep -q .; then
echo 'Using preinstalled Playwright Chromium from '"\${PW_BROWSER_PATH}"
else
browser_ok=false &&
for i in 1 2 3; do
echo 'Playwright browser install attempt' "\$i"'/3' &&
if timeout 1800 yarn playwright install chromium; then
browser_ok=true
break
fi
if [ "\$i" -lt 3 ]; then
echo 'Playwright install attempt failed; retrying in 20s'
sleep 20
fi
done &&
if [ "\$browser_ok" != 'true' ]; then
echo '❌ Playwright browser install failed after 3 attempts'
exit 1
fi
fi &&
yarn test:e2e --reporter=list --timeout=90000
else
echo 'No E2E tests found'
fi
" 2>&1 | tee "${LOG_FILE}"
TEST_STATUS=${PIPESTATUS[0]}
if [ "${TEST_STATUS}" -ne 0 ]; then
echo "❌ E2E tests failed (exit=${TEST_STATUS})"
echo "--- Last 200 lines of E2E output ---"
tail -n 200 "${LOG_FILE}" || true
exit "${TEST_STATUS}"
fi
- *failure_diagnostics_step