From f0b259e1c30d8905ab843cd9736d5926c308d6cf Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Tue, 7 Jul 2026 18:01:48 -0400 Subject: [PATCH] Add tester image lanes and production gates to CICD workflow --- .gitea/workflows/cicd.yaml | 301 +++++++++++++++++++++++++++++----- Dockerfile.e2e-tester | 7 + Dockerfile.integration-tester | 9 + 3 files changed, 273 insertions(+), 44 deletions(-) create mode 100644 Dockerfile.e2e-tester create mode 100644 Dockerfile.integration-tester diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index d65abbc..a478d6c 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -275,7 +275,6 @@ jobs: backend-tests: name: Backend Tests - # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act timeout-minutes: 25 needs: build_cicd @@ -485,10 +484,221 @@ jobs: - *failure_diagnostics_step + cicd-tests-complete: + name: CICD Tests Complete + runs-on: ubuntu-act + needs: [backend-tests, precommit-tests, frontend-tests, xdoctest] + steps: + - name: Confirm all source lanes passed + run: | + echo "✅ Source checks complete" + + build-integration-tester-image: + name: Build Integration Tester Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} + integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for integration tester image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.integration-tester + else + git checkout HEAD -- Dockerfile.integration-tester + fi + + - name: Build and push integration tester image + id: integration_tester_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration" + INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.integration-tester -t plex-playlist-integration:"${HEAD_SHA}" . + docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}" + + retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4 + retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3 + + INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then + echo "❌ Unable to resolve integration tester digest" + exit 1 + fi + + echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + + build-e2e-tester-image: + name: Build E2E Tester Image + runs-on: ubuntu-act-8gb + needs: [build_cicd, cicd-tests-complete] + outputs: + e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} + e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} + steps: + - name: Resolve head SHA + id: meta + run: | + echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT" + + - name: Minimal checkout for E2E tester image inputs + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f ~/.ssh/id_rsa' EXIT + + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null + + GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + + if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then + git checkout FETCH_HEAD -- Dockerfile.e2e-tester + else + git checkout HEAD -- Dockerfile.e2e-tester + fi + + - name: Build and push E2E tester image + id: e2e_tester_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e" + E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}" + + docker build -f Dockerfile.e2e-tester -t plex-playlist-e2e:"${HEAD_SHA}" . + docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}" + + retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4 + retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3 + + E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then + echo "❌ Unable to resolve E2E tester digest" + exit 1 + fi + + echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step + runtime_images: name: Build and Publish Runtime Images runs-on: ubuntu-act-8gb - needs: [build_cicd, precommit-tests, backend-tests, frontend-tests, xdoctest] + needs: [build_cicd, cicd-tests-complete] timeout-minutes: 45 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} @@ -693,6 +903,15 @@ jobs: - *failure_diagnostics_step + production-images-complete: + name: Production Images Complete + runs-on: ubuntu-act + needs: runtime_images + steps: + - name: Confirm deployable runtime images are published + run: | + echo "✅ Deployable backend/frontend runtime images are available" + runtime-images-postmortem: name: Runtime Images Failure Postmortem # Run on the broader runner pool so we can still capture diagnostics when @@ -916,7 +1135,7 @@ jobs: # Pin integration tests to high-memory worker to reduce setup-stage runner churn. runs-on: ubuntu-act-8gb timeout-minutes: 20 - needs: runtime_images + needs: [runtime_images, production-images-complete, build-integration-tester-image] steps: - name: Identify runner run: | @@ -936,6 +1155,7 @@ jobs: HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }} DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }} DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} run: | set -euo pipefail set -o pipefail @@ -986,50 +1206,37 @@ jobs: output_file="$2" probe_tmp="$(mktemp)" - run_probe_with_python() { - python_bin="$1" - docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend "${python_bin}" -c "$(printf '%s\n' \ - 'import sys' \ - 'import urllib.error' \ - 'import urllib.request' \ - 'endpoint_path = sys.argv[1]' \ - 'url = f"http://127.0.0.1:8000{endpoint_path}"' \ - 'try:' \ - ' with urllib.request.urlopen(url, timeout=2) as response:' \ - ' body = response.read().decode("utf-8", errors="replace")' \ - ' print(response.status)' \ - ' print(body)' \ - 'except urllib.error.HTTPError as err:' \ - ' body = err.read().decode("utf-8", errors="replace")' \ - ' print(err.code)' \ - ' print(body)' \ - 'except Exception:' \ - ' print("000")' \ - ' print("")' - )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null - ' print("")')" "${endpoint_path}" - } - - probe_err="$(mktemp)" - - if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then - : - else + if ! docker run --rm \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + "${INTEGRATION_TESTER_DIGEST_REF}" \ + python -c "$(printf '%s\n' \ + 'import sys' \ + 'import urllib.error' \ + 'import urllib.request' \ + 'endpoint_path = sys.argv[1]' \ + 'url = f"http://backend:8000{endpoint_path}"' \ + 'try:' \ + ' with urllib.request.urlopen(url, timeout=2) as response:' \ + ' body = response.read().decode("utf-8", errors="replace")' \ + ' print(response.status)' \ + ' print(body)' \ + 'except urllib.error.HTTPError as err:' \ + ' body = err.read().decode("utf-8", errors="replace")' \ + ' print(err.code)' \ + ' print(body)' \ + 'except Exception:' \ + ' print("000")' \ + ' print("")')" \ + "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then : >"${output_file}" - { - echo '{"status":"probe_exec_failed"}' - sed -n '1,5p' "${probe_err}" || true - } >"${output_file}" - rm -f "${probe_tmp}" - rm -f "${probe_err}" echo "000" + rm -f "${probe_tmp}" return 0 fi http_code="$(head -n 1 "${probe_tmp}")" tail -n +2 "${probe_tmp}" >"${output_file}" rm -f "${probe_tmp}" - rm -f "${probe_err}" echo "${http_code}" } @@ -1163,7 +1370,7 @@ jobs: # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act-8gb timeout-minutes: 45 - needs: runtime_images + needs: [runtime_images, production-images-complete, build-integration-tester-image, build-e2e-tester-image] steps: - name: Identify runner run: | @@ -1186,6 +1393,8 @@ jobs: DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }} DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_frontend_tag_ref }} DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_frontend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }} + E2E_TESTER_DIGEST_REF: ${{ needs.build-e2e-tester-image.outputs.e2e_tester_digest_ref }} run: | set -euo pipefail set -o pipefail @@ -1295,7 +1504,10 @@ jobs: output_file="$2" probe_tmp="$(mktemp)" - if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \ + if ! docker run --rm \ + --network "${COMPOSE_PROJECT_NAME}_default" \ + "${INTEGRATION_TESTER_DIGEST_REF}" \ + python -c "$(printf '%s\n' \ 'import sys' \ 'import urllib.error' \ 'import urllib.request' \ @@ -1311,7 +1523,8 @@ jobs: ' print(body)' \ 'except Exception:' \ ' print(\"000\")' \ - ' print(\"\")')" "${target_url}" >"${probe_tmp}" 2>/dev/null; then + ' print("")')" \ + "${target_url}" >"${probe_tmp}" 2>/dev/null; then : >"${output_file}" rm -f "${probe_tmp}" echo "000" @@ -1488,8 +1701,8 @@ jobs: -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \ -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \ -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \ - mcr.microsoft/playwright:v1.56.1-jammy \ - bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \ + "${E2E_TESTER_DIGEST_REF}" \ + bash -lc "cd /workspace/frontend && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \ 2>&1 | tee "${LOG_FILE}" TEST_STATUS=${PIPESTATUS[0]} diff --git a/Dockerfile.e2e-tester b/Dockerfile.e2e-tester new file mode 100644 index 0000000..57fff55 --- /dev/null +++ b/Dockerfile.e2e-tester @@ -0,0 +1,7 @@ +FROM mcr.microsoft/playwright:v1.56.1-jammy + +WORKDIR /workspace/frontend + +RUN corepack enable + +CMD ["bash"] diff --git a/Dockerfile.integration-tester b/Dockerfile.integration-tester new file mode 100644 index 0000000..989ef9d --- /dev/null +++ b/Dockerfile.integration-tester @@ -0,0 +1,9 @@ +FROM python:3.14-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +CMD ["bash"]