Add tester image lanes and production gates to CICD workflow

.gitea/workflows/cicd.yaml

@@ -275,7 +275,6 @@ jobs:
 
   backend-tests:
     name: Backend Tests
-    # Use ubuntu-act runner pool for consistent availability.
     runs-on: ubuntu-act
     timeout-minutes: 25
     needs: build_cicd
@@ -485,10 +484,221 @@ jobs:
 
       - *failure_diagnostics_step
 
+  cicd-tests-complete:
+    name: CICD Tests Complete
+    runs-on: ubuntu-act
+    needs: [backend-tests, precommit-tests, frontend-tests, xdoctest]
+    steps:
+      - name: Confirm all source lanes passed
+        run: |
+          echo "✅ Source checks complete"
+
+  build-integration-tester-image:
+    name: Build Integration Tester Image
+    runs-on: ubuntu-act-8gb
+    needs: [build_cicd, cicd-tests-complete]
+    outputs:
+      integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }}
+      integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }}
+    steps:
+      - name: Resolve head SHA
+        id: meta
+        run: |
+          echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT"
+
+      - name: Minimal checkout for integration tester image inputs
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
+        run: |
+          set -euo pipefail
+          umask 077
+          trap 'rm -f ~/.ssh/id_rsa' EXIT
+
+          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
+            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
+          fi
+
+          mkdir -p ~/.ssh
+          echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
+
+          GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
+            git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
+
+          if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
+            git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
+            git checkout FETCH_HEAD -- Dockerfile.integration-tester
+          else
+            git checkout HEAD -- Dockerfile.integration-tester
+          fi
+
+      - name: Build and push integration tester image
+        id: integration_tester_ref
+        env:
+          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
+        run: |
+          set -euo pipefail
+
+          retry_registry_op() {
+            op_name="$1"
+            image_ref="$2"
+            attempts="${3:-5}"
+            backoff="${4:-3}"
+            attempt=1
+
+            while [ "${attempt}" -le "${attempts}" ]; do
+              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
+              if [ "${op_name}" = "push" ]; then
+                if docker push "${image_ref}"; then
+                  return 0
+                fi
+              else
+                if docker pull "${image_ref}" >/dev/null; then
+                  return 0
+                fi
+              fi
+
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "Retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+
+            return 1
+          }
+
+          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration"
+          INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}"
+
+          docker build -f Dockerfile.integration-tester -t plex-playlist-integration:"${HEAD_SHA}" .
+          docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}"
+
+          retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4
+          retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3
+
+          INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)"
+          if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then
+            echo "❌ Unable to resolve integration tester digest"
+            exit 1
+          fi
+
+          echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
+          echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
+
+      - *failure_diagnostics_step
+
+  build-e2e-tester-image:
+    name: Build E2E Tester Image
+    runs-on: ubuntu-act-8gb
+    needs: [build_cicd, cicd-tests-complete]
+    outputs:
+      e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }}
+      e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }}
+    steps:
+      - name: Resolve head SHA
+        id: meta
+        run: |
+          echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT"
+
+      - name: Minimal checkout for E2E tester image inputs
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
+        run: |
+          set -euo pipefail
+          umask 077
+          trap 'rm -f ~/.ssh/id_rsa' EXIT
+
+          if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
+            echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
+          fi
+
+          mkdir -p ~/.ssh
+          echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
+
+          GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
+            git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
+
+          if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
+            git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
+            git checkout FETCH_HEAD -- Dockerfile.e2e-tester
+          else
+            git checkout HEAD -- Dockerfile.e2e-tester
+          fi
+
+      - name: Build and push E2E tester image
+        id: e2e_tester_ref
+        env:
+          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
+        run: |
+          set -euo pipefail
+
+          retry_registry_op() {
+            op_name="$1"
+            image_ref="$2"
+            attempts="${3:-5}"
+            backoff="${4:-3}"
+            attempt=1
+
+            while [ "${attempt}" -le "${attempts}" ]; do
+              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
+              if [ "${op_name}" = "push" ]; then
+                if docker push "${image_ref}"; then
+                  return 0
+                fi
+              else
+                if docker pull "${image_ref}" >/dev/null; then
+                  return 0
+                fi
+              fi
+
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "Retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+
+            return 1
+          }
+
+          echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
+          E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e"
+          E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}"
+
+          docker build -f Dockerfile.e2e-tester -t plex-playlist-e2e:"${HEAD_SHA}" .
+          docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}"
+
+          retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4
+          retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3
+
+          E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)"
+          if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then
+            echo "❌ Unable to resolve E2E tester digest"
+            exit 1
+          fi
+
+          echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
+          echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
+
+      - *failure_diagnostics_step
+
   runtime_images:
     name: Build and Publish Runtime Images
     runs-on: ubuntu-act-8gb
-    needs: [build_cicd, precommit-tests, backend-tests, frontend-tests, xdoctest]
+    needs: [build_cicd, cicd-tests-complete]
     timeout-minutes: 45
     outputs:
       head_sha: ${{ steps.meta.outputs.head_sha }}
@@ -693,6 +903,15 @@ jobs:
 
       - *failure_diagnostics_step
 
+  production-images-complete:
+    name: Production Images Complete
+    runs-on: ubuntu-act
+    needs: runtime_images
+    steps:
+      - name: Confirm deployable runtime images are published
+        run: |
+          echo "✅ Deployable backend/frontend runtime images are available"
+
   runtime-images-postmortem:
     name: Runtime Images Failure Postmortem
     # Run on the broader runner pool so we can still capture diagnostics when
@@ -916,7 +1135,7 @@ jobs:
     # Pin integration tests to high-memory worker to reduce setup-stage runner churn.
     runs-on: ubuntu-act-8gb
     timeout-minutes: 20
-    needs: runtime_images
+    needs: [runtime_images, production-images-complete, build-integration-tester-image]
     steps:
       - name: Identify runner
         run: |
@@ -936,6 +1155,7 @@ jobs:
           HEAD_SHA: ${{ needs.runtime_images.outputs.head_sha }}
           DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_backend_tag_ref }}
           DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }}
+          INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }}
         run: |
           set -euo pipefail
           set -o pipefail
@@ -986,50 +1206,37 @@ jobs:
             output_file="$2"
             probe_tmp="$(mktemp)"
 
-            run_probe_with_python() {
-              python_bin="$1"
-              docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend "${python_bin}" -c "$(printf '%s\n' \
-              'import sys' \
-              'import urllib.error' \
-              'import urllib.request' \
-              'endpoint_path = sys.argv[1]' \
-              'url = f"http://127.0.0.1:8000{endpoint_path}"' \
-              'try:' \
-              '    with urllib.request.urlopen(url, timeout=2) as response:' \
-              '        body = response.read().decode("utf-8", errors="replace")' \
-              '        print(response.status)' \
-              '        print(body)' \
-              'except urllib.error.HTTPError as err:' \
-              '    body = err.read().decode("utf-8", errors="replace")' \
-              '    print(err.code)' \
-              '    print(body)' \
-              'except Exception:' \
-              '    print("000")' \
-              '    print("")'
-            )" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null
-              '    print("")')" "${endpoint_path}"
-            }
-
-            probe_err="$(mktemp)"
-
-            if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then
-              :
-            else
+            if ! docker run --rm \
+              --network "${COMPOSE_PROJECT_NAME}_default" \
+              "${INTEGRATION_TESTER_DIGEST_REF}" \
+              python -c "$(printf '%s\n' \
+                'import sys' \
+                'import urllib.error' \
+                'import urllib.request' \
+                'endpoint_path = sys.argv[1]' \
+                'url = f"http://backend:8000{endpoint_path}"' \
+                'try:' \
+                '    with urllib.request.urlopen(url, timeout=2) as response:' \
+                '        body = response.read().decode("utf-8", errors="replace")' \
+                '        print(response.status)' \
+                '        print(body)' \
+                'except urllib.error.HTTPError as err:' \
+                '    body = err.read().decode("utf-8", errors="replace")' \
+                '    print(err.code)' \
+                '    print(body)' \
+                'except Exception:' \
+                '    print("000")' \
+                '    print("")')" \
+              "${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then
               : >"${output_file}"
-              {
-                echo '{"status":"probe_exec_failed"}'
-                sed -n '1,5p' "${probe_err}" || true
-              } >"${output_file}"
-              rm -f "${probe_tmp}"
-              rm -f "${probe_err}"
               echo "000"
+              rm -f "${probe_tmp}"
               return 0
             fi
 
             http_code="$(head -n 1 "${probe_tmp}")"
             tail -n +2 "${probe_tmp}" >"${output_file}"
             rm -f "${probe_tmp}"
-            rm -f "${probe_err}"
             echo "${http_code}"
           }
 
@@ -1163,7 +1370,7 @@ jobs:
     # Use ubuntu-act runner pool for consistent availability.
     runs-on: ubuntu-act-8gb
     timeout-minutes: 45
-    needs: runtime_images
+    needs: [runtime_images, production-images-complete, build-integration-tester-image, build-e2e-tester-image]
     steps:
       - name: Identify runner
         run: |
@@ -1186,6 +1393,8 @@ jobs:
           DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_backend_digest_ref }}
           DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs.runtime_images.outputs.deployable_frontend_tag_ref }}
           DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs.runtime_images.outputs.deployable_frontend_digest_ref }}
+          INTEGRATION_TESTER_DIGEST_REF: ${{ needs.build-integration-tester-image.outputs.integration_tester_digest_ref }}
+          E2E_TESTER_DIGEST_REF: ${{ needs.build-e2e-tester-image.outputs.e2e_tester_digest_ref }}
         run: |
           set -euo pipefail
           set -o pipefail
@@ -1295,7 +1504,10 @@ jobs:
             output_file="$2"
             probe_tmp="$(mktemp)"
 
-            if ! docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \
+            if ! docker run --rm \
+              --network "${COMPOSE_PROJECT_NAME}_default" \
+              "${INTEGRATION_TESTER_DIGEST_REF}" \
+              python -c "$(printf '%s\n' \
               'import sys' \
               'import urllib.error' \
               'import urllib.request' \
@@ -1311,7 +1523,8 @@ jobs:
               '    print(body)' \
               'except Exception:' \
               '    print(\"000\")' \
-              '    print(\"\")')" "${target_url}" >"${probe_tmp}" 2>/dev/null; then
+              '    print("")')" \
+              "${target_url}" >"${probe_tmp}" 2>/dev/null; then
               : >"${output_file}"
               rm -f "${probe_tmp}"
               echo "000"
@@ -1488,8 +1701,8 @@ jobs:
             -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \
             -v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \
             -v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \
-            mcr.microsoft/playwright:v1.56.1-jammy \
-            bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \
+            "${E2E_TESTER_DIGEST_REF}" \
+            bash -lc "cd /workspace/frontend && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000" \
             2>&1 | tee "${LOG_FILE}"
 
           TEST_STATUS=${PIPESTATUS[0]}

Dockerfile.e2e-tester

@@ -0,0 +1,7 @@
+FROM mcr.microsoft/playwright:v1.56.1-jammy
+
+WORKDIR /workspace/frontend
+
+RUN corepack enable
+
+CMD ["bash"]

Dockerfile.integration-tester

@@ -0,0 +1,9 @@
+FROM python:3.14-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workspace
+
+CMD ["bash"]