From c996d92406216b05bb38a553869027d09138070f Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 2 Jul 2026 08:47:42 -0400 Subject: [PATCH] feat(ci): replace integration lane with runtime black-box tests --- .gitea/workflows/cicd-tests.yaml | 173 ++++++++++++++++++++++-- .gitea/workflows/docker-build-main.yaml | 44 ++++++ docs/CICD_SUCCESS_SUMMARY.md | 4 + docs/DEPLOYABLE_RUNTIME_CONTRACT.md | 6 + 4 files changed, 212 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index de807a8..315ec50 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -6,6 +6,12 @@ on: head_sha: description: Commit SHA to process required: false + deployable_backend_tag_ref: + description: Deployable backend image tag reference + required: false + deployable_backend_digest_ref: + description: Deployable backend image digest reference + required: false source_workflow: description: Upstream workflow name required: false @@ -30,6 +36,8 @@ jobs: timeout-minutes: 8 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner run: | @@ -48,6 +56,8 @@ jobs: EVENT_NAME: ${{ github.event_name }} SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }} HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} REF: ${{ github.ref }} REF_NAME: ${{ github.ref_name }} @@ -61,6 +71,8 @@ jobs: echo "source_workflow=${SOURCE_WORKFLOW}" echo "head_sha_input=${HEAD_SHA_INPUT}" echo "head_sha=${RESOLVED_HEAD_SHA}" + echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" + echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" echo "ref=${REF}" echo "ref_name=${REF_NAME}" echo "head_ref=${HEAD_REF}" @@ -70,10 +82,14 @@ jobs: id: meta env: HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }} HEAD_SHA_FALLBACK: ${{ github.sha }} run: | RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}" echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT" - &failure_diagnostics_step name: Failure diagnostics @@ -269,7 +285,7 @@ jobs: - *failure_diagnostics_step integration-tests: - name: Integration Tests + name: Runtime Black-Box Integration Tests # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 @@ -288,28 +304,155 @@ jobs: echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - *configure_registry_host_step - - *ensure_cicd_image_step - - name: Run integration tests + - name: Run runtime black-box integration checks env: - HEAD_SHA: ${{ needs.setup.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }} run: | + set -euo pipefail set -o pipefail - LOG_FILE="$(mktemp)" - docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c " - cd /workspace/backend && - source .venv/bin/activate && - if [ -d 'tests/integration' ]; then - uv run pytest tests/integration/ -v --tb=short - else - echo 'No integration tests found' + LOG_FILE="$(mktemp)" + NETWORK_NAME="plex-blackbox-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + DB_CONTAINER="plex-blackbox-db-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + BACKEND_CONTAINER="plex-blackbox-backend-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + DB_PASSWORD="plex_password" + DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist" + + dump_failure_context() { + echo "=== Runtime Integration Failure Context ===" + docker ps -a || true + echo "--- Backend logs (tail 200) ---" + docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Database logs (tail 200) ---" + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true + echo "--- Backend inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true + echo "--- Database inspect (status/image) ---" + docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true + } + + cleanup() { + docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true + docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true + docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true + } + + trap cleanup EXIT + + { + if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Missing deployable backend image references from dispatch inputs" + exit 1 fi - " 2>&1 | tee "${LOG_FILE}" + + echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin + + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" + + DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}" + EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}" + ACTUAL_TAG_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then + echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}" + exit 1 + fi + + if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then + echo "❌ Tag and digest mismatch" + echo "expected=${EXPECTED_DIGEST_REF}" + echo "actual=${ACTUAL_TAG_DIGEST_REF}" + exit 1 + fi + + echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}" + echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}" + + docker network create "${NETWORK_NAME}" + + docker run -d \ + --name "${DB_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -e POSTGRES_DB=plex_playlist \ + -e POSTGRES_USER=plex_user \ + -e POSTGRES_PASSWORD="${DB_PASSWORD}" \ + postgres:16-alpine + + db_ready=false + for i in $(seq 1 30); do + if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then + db_ready=true + break + fi + sleep 2 + done + if [ "${db_ready}" != "true" ]; then + echo "❌ Database did not become ready" + dump_failure_context + exit 1 + fi + + docker run -d \ + --name "${BACKEND_CONTAINER}" \ + --network "${NETWORK_NAME}" \ + -p 18000:8000 \ + -e DATABASE_URL="${DB_URL}" \ + -e ENVIRONMENT=production \ + "${EXPECTED_DIGEST_REF}" + + backend_ready=false + for i in $(seq 1 40); do + health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" + if [ "${health_code}" = "200" ]; then + backend_ready=true + break + fi + sleep 2 + done + if [ "${backend_ready}" != "true" ]; then + echo "❌ Backend did not become healthy" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + root_code="$(curl -sS -o /tmp/blackbox-root.json -w "%{http_code}" "http://127.0.0.1:18000/" || true)" + if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then + echo "❌ Root endpoint validation failed" + cat /tmp/blackbox-root.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + compatibility_code="$(curl -sS -o /tmp/blackbox-compatibility.json -w "%{http_code}" "http://127.0.0.1:18000/compatibility" || true)" + if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then + echo "❌ Compatibility endpoint validation failed" + cat /tmp/blackbox-compatibility.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + health_code="$(curl -sS -o /tmp/blackbox-health.json -w "%{http_code}" "http://127.0.0.1:18000/health" || true)" + if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then + echo "❌ Health endpoint validation failed" + cat /tmp/blackbox-health.json 2>/dev/null || true + dump_failure_context + exit 1 + fi + + echo "✅ Runtime black-box integration checks passed" + } 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} if [ "${TEST_STATUS}" -ne 0 ]; then - echo "❌ Integration tests failed (exit=${TEST_STATUS})" - echo "--- Last 200 lines of integration test output ---" + echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})" + echo "--- Last 200 lines of runtime black-box integration output ---" tail -n 200 "${LOG_FILE}" || true exit "${TEST_STATUS}" fi diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml index 5ff3303..bdf21c2 100644 --- a/.gitea/workflows/docker-build-main.yaml +++ b/.gitea/workflows/docker-build-main.yaml @@ -138,6 +138,8 @@ jobs: timeout-minutes: 60 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} steps: - name: Identify runner @@ -251,6 +253,44 @@ jobs: --image deployable-frontend:"${HEAD_SHA}" \ --profile frontend + - name: Push deployable backend runtime image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker push "${DEPLOYABLE_BACKEND_TAG_REF}" + docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \ + | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \ + | head -n 1 + } || true)" + + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve deployable backend digest reference" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" + - name: Build and push complete CICD image env: PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} @@ -347,6 +387,8 @@ jobs: ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }} PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} HEAD_SHA: ${{ needs.build.outputs.head_sha }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }} REPO_FULL: ${{ github.repository }} HEAD_REF: ${{ github.head_ref }} REF_NAME: ${{ github.ref_name }} @@ -426,6 +468,8 @@ jobs: --head-sha "${HEAD_SHA}" --source-workflow "CICD Main Build" --trace-id "${TRACE_ID}" + --input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" + --input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" ) for API_BASE in "${CANDIDATE_API_BASES[@]}"; do diff --git a/docs/CICD_SUCCESS_SUMMARY.md b/docs/CICD_SUCCESS_SUMMARY.md index de4902b..a9decc3 100644 --- a/docs/CICD_SUCCESS_SUMMARY.md +++ b/docs/CICD_SUCCESS_SUMMARY.md @@ -33,6 +33,8 @@ - **E2E Tests**: Simplified Docker approach matching other successful test patterns - **Playwright**: Chromium-only CI strategy (95%+ browser market coverage) - **Registry Operations**: Consistent approach across all test phases +- **Backend Runtime Integration**: Black-box checks run against started deployable + backend containers with commit-tag and digest pinning ## 🛠️ **Critical Issues Resolved** @@ -82,6 +84,8 @@ Frontend Environment (Yarn PnP) ↓ (state regeneration) Test Execution (all phases) ↓ (consistent Docker approach) +Runtime Black-Box Integration + ↓ (deployable backend tag+digest verification) E2E Testing (Playwright) ↓ (Chromium + network resilience) ✅ SUCCESS diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index 64ddf00..1d9e999 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -157,6 +157,12 @@ Current enforcement implemented in CI: - Workflow: `.gitea/workflows/docker-build-main.yaml` - Checks include binary presence and profile-specific package metadata probes to detect CI/development tooling leakage. +- Runtime black-box integration checks against deployed backend container: + - Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job) + - Inputs: deployable backend commit tag reference and immutable digest + reference from main build dispatch. + - Assertions: digest/tag consistency and live endpoint behavior for `/`, + `/compatibility`, and `/health` on started runtime containers. Future follow-up automation under epic #66 may expand this with: