47214027da
Some checks failed
CICD / Build and Publish CICD Base Image (push) Successful in 1m48s
CICD / Build and Push CICD Image (push) Successful in 49m2s
CICD / Build CICD Image Failure Postmortem (push) Has been skipped
CICD / Frontend Tests (push) Successful in 1m5s
CICD / Backend Doctests (push) Successful in 37s
CICD / Pre-commit Checks (push) Successful in 10m17s
CICD / Backend Tests (push) Successful in 15m2s
CICD / Source Lanes Failure Postmortem (push) Has been skipped
CICD / CICD Tests Complete (push) Successful in 3s
CICD / Build Integration Tester Image (push) Successful in 1m28s
CICD / Build Backend Base Image (push) Successful in 2m24s
CICD / Build Backend Main Image (push) Successful in 1m31s
CICD / Build Frontend Base Image (push) Successful in 6m16s
CICD / Build and Publish Runtime Images (Legacy Disabled) (push) Has been skipped
CICD / Build E2E Tester Image (push) Successful in 7m16s
CICD / Build Frontend Main Image (push) Successful in 5m43s
CICD / Runtime Images Failure Postmortem (push) Has been skipped
CICD / Production Images Complete (push) Successful in 8s
CICD / Runtime Black-Box Integration Tests (push) Successful in 3m13s
CICD / Integration Tests Failure Postmortem (push) Has been skipped
CICD / End-to-End Tests (push) Failing after 25m2s
CICD / E2E Tests Failure Postmortem (push) Successful in 10s
2648 lines
103 KiB
YAML
2648 lines
103 KiB
YAML
name: CICD
|
|
|
|
on:
|
|
push:
|
|
pull_request:
|
|
branches: [main, develop]
|
|
workflow_dispatch:
|
|
inputs:
|
|
head_sha:
|
|
description: Commit SHA to process
|
|
required: false
|
|
base_hash:
|
|
description: Immutable base hash to use for CICD base image
|
|
required: false
|
|
force_rebuild_base:
|
|
description: Force CICD base rebuild
|
|
required: false
|
|
default: "false"
|
|
trace_id:
|
|
description: Correlation id propagated across CICD jobs
|
|
required: false
|
|
|
|
env:
|
|
GITEA_SSH_HOST: kankali.darkhelm.lan
|
|
GITEA_SSH_PORT: "2222"
|
|
GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
|
|
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
|
GITEA_REGISTRY_IP: 10.18.75.2
|
|
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
|
CI_PRUNE_AT_JOB_START: "true"
|
|
CI_PRUNE_THRESHOLD_PERCENT: "90"
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
concurrency:
|
|
group: cicd-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
publish-base:
|
|
name: Build and Publish CICD Base Image
|
|
runs-on: ubuntu-act-8gb
|
|
timeout-minutes: 35
|
|
outputs:
|
|
base_hash: ${{ steps.base-state.outputs.base_hash }}
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
steps:
|
|
- &identify_runner_step
|
|
name: Identify runner
|
|
run: |
|
|
echo "=== Runner Identity ==="
|
|
echo "runner_name=${RUNNER_NAME:-}"
|
|
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
|
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
|
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
|
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
|
echo "runner_os=${RUNNER_OS:-unknown}"
|
|
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
|
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
- &pre_job_prune_step
|
|
name: Pre-job Docker prune
|
|
run: |
|
|
if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then
|
|
echo "Skipping pre-job prune"
|
|
exit 0
|
|
fi
|
|
|
|
used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
|
|
echo "disk_used_before=${used_before}%"
|
|
echo "prune_threshold=${threshold}%"
|
|
|
|
if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
|
|
docker builder prune -af || true
|
|
docker system prune -af --volumes || true
|
|
used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
echo "disk_used_after=${used_after}%"
|
|
else
|
|
echo "Skipping prune; usage below threshold"
|
|
fi
|
|
|
|
- name: Resolve head SHA
|
|
id: meta
|
|
env:
|
|
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
|
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
|
run: |
|
|
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
|
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Minimal checkout for base inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh
|
|
else
|
|
git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh
|
|
fi
|
|
|
|
chmod +x scripts/compute-cicd-base-hash.sh
|
|
|
|
- name: Compute base hash and inspect registry
|
|
id: base-state
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
FORCE_REBUILD: ${{ github.event.inputs.force_rebuild_base }}
|
|
run: |
|
|
set -euo pipefail
|
|
BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
|
|
BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
|
|
BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest"
|
|
|
|
echo "base_hash=${BASE_HASH}" >> "$GITHUB_OUTPUT"
|
|
echo "base_ref_hash=${BASE_REF_HASH}" >> "$GITHUB_OUTPUT"
|
|
echo "base_ref_latest=${BASE_REF_LATEST}" >> "$GITHUB_OUTPUT"
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null
|
|
|
|
if [ "${FORCE_REBUILD:-false}" = "true" ]; then
|
|
echo "needs_build=true" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
|
|
if timeout 900 docker pull "${BASE_REF_HASH}" >/dev/null 2>&1; then
|
|
echo "needs_build=false" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "needs_build=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Build and push base image
|
|
if: steps.base-state.outputs.needs_build == 'true'
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
BASE_HASH: ${{ steps.base-state.outputs.base_hash }}
|
|
BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }}
|
|
BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
|
|
PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
|
|
retry_registry_op pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 5 3
|
|
|
|
PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}")
|
|
|
|
docker build --progress=plain -f Dockerfile.cicd-base \
|
|
--build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \
|
|
--build-arg BASE_IMAGE_HASH="${BASE_HASH}" \
|
|
--build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \
|
|
-t cicd-base:latest .
|
|
|
|
docker tag cicd-base:latest "${BASE_REF_HASH}"
|
|
docker tag cicd-base:latest "${BASE_REF_LATEST}"
|
|
retry_registry_op push "${BASE_REF_HASH}" 5 4
|
|
retry_registry_op push "${BASE_REF_LATEST}" 5 4
|
|
|
|
- &failure_diagnostics_step
|
|
name: Failure diagnostics
|
|
if: failure()
|
|
run: |
|
|
echo "=== Failure Diagnostics ==="
|
|
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
|
echo "runner_name=${RUNNER_NAME:-unknown}"
|
|
echo "runner_hostname=${HOSTNAME:-unknown}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
ps aux --sort=-%mem | head -n 30 || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
echo "=== Docker Diagnostics ==="
|
|
docker version || true
|
|
docker info || true
|
|
docker ps -a || true
|
|
docker images --digests | head -n 50 || true
|
|
fi
|
|
dmesg | tail -n 120 || true
|
|
|
|
build_cicd:
|
|
name: Build and Push CICD Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: publish-base
|
|
timeout-minutes: 60
|
|
outputs:
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
steps:
|
|
- name: Resolve head SHA
|
|
id: meta
|
|
run: |
|
|
echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *pre_job_prune_step
|
|
|
|
- name: Minimal checkout for CICD build inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh
|
|
else
|
|
git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base scripts/compute-cicd-base-hash.sh
|
|
fi
|
|
chmod +x scripts/compute-cicd-base-hash.sh
|
|
|
|
- name: Build and push complete CICD image
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }}
|
|
BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f /tmp/ssh_key' EXIT
|
|
|
|
trim_spaces() {
|
|
printf '%s' "$1" | tr -d '[:space:]'
|
|
}
|
|
|
|
is_hex() {
|
|
value="$1"
|
|
if [ -z "${value}" ]; then
|
|
return 1
|
|
fi
|
|
case "${value}" in
|
|
(*[!0-9a-fA-F]*) return 1 ;;
|
|
(*) return 0 ;;
|
|
esac
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
echo "=== Pre-build disk telemetry ==="
|
|
df -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker system df || true
|
|
docker builder prune -af || true
|
|
docker system prune -af --volumes || true
|
|
echo "=== Post-prune disk telemetry ==="
|
|
df -h || true
|
|
docker system df || true
|
|
fi
|
|
|
|
ensure_registry_auth_realm_host
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
|
|
RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")"
|
|
if ! is_hex "${RESOLVED_HEAD_SHA}"; then
|
|
echo "❌ Invalid or empty HEAD_SHA resolved for build_cicd: '${HEAD_SHA:-}'"
|
|
exit 1
|
|
fi
|
|
|
|
BASE_HASH="$(trim_spaces "${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}")"
|
|
if ! is_hex "${BASE_HASH}"; then
|
|
BASE_HASH="$(./scripts/compute-cicd-base-hash.sh | tr -d '[:space:]')"
|
|
fi
|
|
|
|
if ! is_hex "${BASE_HASH}"; then
|
|
echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_INPUT:-${BASE_HASH_FROM_BUILD}}'"
|
|
exit 1
|
|
fi
|
|
|
|
echo "resolved_head_sha=${RESOLVED_HEAD_SHA}"
|
|
echo "resolved_base_hash=${BASE_HASH}"
|
|
|
|
BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
|
|
retry_registry_op pull "${BASE_IMAGE}" 5 3
|
|
|
|
echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
|
|
chmod 600 /tmp/ssh_key
|
|
|
|
docker build -f Dockerfile.cicd \
|
|
--secret id=ssh_private_key,src=/tmp/ssh_key \
|
|
--add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \
|
|
--build-arg GITHUB_SHA="${RESOLVED_HEAD_SHA}" \
|
|
--build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \
|
|
-t cicd:latest .
|
|
|
|
CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest"
|
|
CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}"
|
|
|
|
docker tag cicd:latest "${CICD_LATEST_REF}"
|
|
docker tag cicd:latest "${CICD_SHA_REF}"
|
|
retry_registry_op push "${CICD_LATEST_REF}" 5 4
|
|
retry_registry_op push "${CICD_SHA_REF}" 5 4
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-cicd-postmortem:
|
|
name: Build CICD Image Failure Postmortem
|
|
runs-on: ubuntu-act
|
|
needs: build_cicd
|
|
if: always() && needs.build_cicd.result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Identify runner
|
|
run: |
|
|
echo "=== Runner Identity ==="
|
|
echo "runner_name=${RUNNER_NAME:-unknown}"
|
|
echo "runner_hostname=${HOSTNAME:-unknown}"
|
|
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
- name: Build CICD postmortem diagnostics
|
|
run: |
|
|
echo "=== Build CICD Postmortem ==="
|
|
echo "build_cicd_result=${{ needs.build_cicd.result }}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
backend-tests:
|
|
name: Backend Tests
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 25
|
|
needs: build_cicd
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- &configure_registry_host_step
|
|
name: Configure registry host resolution
|
|
run: |
|
|
if [ "${CI_PRUNE_AT_JOB_START:-false}" = "true" ] && command -v docker >/dev/null 2>&1; then
|
|
used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
|
|
echo "disk_used_before=${used_before}%"
|
|
echo "prune_threshold=${threshold}%"
|
|
if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
|
|
docker builder prune -af || true
|
|
docker system prune -af --volumes || true
|
|
used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
echo "disk_used_after=${used_after}%"
|
|
else
|
|
echo "Skipping prune; usage below threshold"
|
|
fi
|
|
fi
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
- &ensure_cicd_image_step
|
|
name: Ensure CICD image is available
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || needs.runtime_images.outputs.head_sha || github.sha }}
|
|
run: |
|
|
IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
|
if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
|
|
echo "Using cached CICD image: ${IMAGE}"
|
|
else
|
|
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
|
|
pulled=false
|
|
for i in 1 2 3; do
|
|
echo "Pull attempt ${i}/3 for ${IMAGE}"
|
|
if docker pull "${IMAGE}"; then
|
|
pulled=true
|
|
break
|
|
fi
|
|
if [ "${i}" -lt 3 ]; then
|
|
sleep_seconds=$((5 * i))
|
|
echo "Pull failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
done
|
|
|
|
if [ "${pulled}" != "true" ]; then
|
|
echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
- name: Run backend tests with coverage
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace/backend &&
|
|
source .venv/bin/activate &&
|
|
uv run pytest -v --tb=short -m 'not integration' --cov=src --cov-report=term-missing --cov-fail-under=95
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Backend tests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of backend test output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
precommit-tests:
|
|
name: Pre-commit Checks
|
|
# Source-level checks should be able to use the full ubuntu-act pool.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 30
|
|
needs: build_cicd
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- *ensure_cicd_image_step
|
|
- name: Run pre-commit checks
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace &&
|
|
if [ ! -d .git ]; then
|
|
git init /workspace
|
|
fi &&
|
|
/workspace/backend/.venv/bin/pre-commit run --all-files --show-diff-on-failure --config .pre-commit-config.yaml
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Pre-commit checks failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of pre-commit output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
frontend-tests:
|
|
name: Frontend Tests
|
|
# Source-level checks should be able to use the full ubuntu-act pool.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 25
|
|
needs: build_cicd
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- *ensure_cicd_image_step
|
|
- name: Run frontend tests with coverage
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace/frontend &&
|
|
yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Frontend tests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of frontend test output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
xdoctest:
|
|
name: Backend Doctests
|
|
# Use ubuntu-act runner pool for consistent availability.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 15
|
|
needs: build_cicd
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- *ensure_cicd_image_step
|
|
- name: Run backend doctests
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace/backend &&
|
|
source .venv/bin/activate &&
|
|
uv run xdoctest src/ --quiet
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Backend doctests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of doctest output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
cicd-tests-complete:
|
|
name: CICD Tests Complete
|
|
runs-on: ubuntu-act
|
|
needs: [backend-tests, precommit-tests, frontend-tests, xdoctest]
|
|
if: always()
|
|
steps:
|
|
- name: Confirm all source lanes passed
|
|
run: |
|
|
set -euo pipefail
|
|
backend_status="${{ needs['backend-tests'].result }}"
|
|
precommit_status="${{ needs['precommit-tests'].result }}"
|
|
frontend_status="${{ needs['frontend-tests'].result }}"
|
|
xdoctest_status="${{ needs.xdoctest.result }}"
|
|
|
|
echo "backend-tests=${backend_status}"
|
|
echo "precommit-tests=${precommit_status}"
|
|
echo "frontend-tests=${frontend_status}"
|
|
echo "xdoctest=${xdoctest_status}"
|
|
|
|
if [ "${backend_status}" != "success" ] || [ "${precommit_status}" != "success" ] || [ "${frontend_status}" != "success" ] || [ "${xdoctest_status}" != "success" ]; then
|
|
echo "❌ One or more CICD source lanes failed"
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Source checks complete"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-backend-base-image:
|
|
name: Build Backend Base Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, cicd-tests-complete]
|
|
outputs:
|
|
backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }}
|
|
backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }}
|
|
steps:
|
|
- &resolve_head_sha_from_build_step
|
|
name: Resolve head SHA
|
|
id: meta
|
|
run: |
|
|
echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Minimal checkout for backend base image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock
|
|
else
|
|
git checkout HEAD -- Dockerfile.backend backend/pyproject.toml backend/uv.lock
|
|
fi
|
|
|
|
- name: Build and push backend base image
|
|
id: backend_base_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base"
|
|
BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}"
|
|
|
|
docker build -f Dockerfile.backend --target dependencies -t plex-playlist-backend-base:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}"
|
|
retry_registry_op push "${BACKEND_BASE_TAG_REF}" 5 4
|
|
retry_registry_op pull "${BACKEND_BASE_TAG_REF}" 5 3
|
|
|
|
BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${BACKEND_BASE_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve backend base image digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-frontend-base-image:
|
|
name: Build Frontend Base Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, cicd-tests-complete]
|
|
outputs:
|
|
frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }}
|
|
frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }}
|
|
steps:
|
|
- *resolve_head_sha_from_build_step
|
|
|
|
- name: Minimal checkout for frontend base image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml
|
|
else
|
|
git checkout HEAD -- Dockerfile.frontend frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml
|
|
fi
|
|
|
|
- name: Build and push frontend base image
|
|
id: frontend_base_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
retry_cmd 5 3 sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin'
|
|
FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base"
|
|
FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}"
|
|
|
|
docker build -f Dockerfile.frontend --target deps -t plex-playlist-frontend-base:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}"
|
|
retry_registry_op push "${FRONTEND_BASE_TAG_REF}" 5 4
|
|
retry_registry_op pull "${FRONTEND_BASE_TAG_REF}" 5 3
|
|
|
|
FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${FRONTEND_BASE_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve frontend base image digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-integration-tester-image:
|
|
name: Build Integration Tester Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, cicd-tests-complete]
|
|
outputs:
|
|
integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }}
|
|
integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }}
|
|
steps:
|
|
- *resolve_head_sha_from_build_step
|
|
|
|
- name: Minimal checkout for integration tester image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- Dockerfile.integration-tester
|
|
else
|
|
git checkout HEAD -- Dockerfile.integration-tester
|
|
fi
|
|
|
|
- name: Build and push integration tester image
|
|
id: integration_tester_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration"
|
|
INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}"
|
|
|
|
docker build -f Dockerfile.integration-tester -t plex-playlist-integration:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}"
|
|
|
|
retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4
|
|
retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3
|
|
|
|
INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve integration tester digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-e2e-tester-image:
|
|
name: Build E2E Tester Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, cicd-tests-complete]
|
|
outputs:
|
|
e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }}
|
|
e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }}
|
|
steps:
|
|
- *resolve_head_sha_from_build_step
|
|
|
|
- name: Minimal checkout for E2E tester image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- Dockerfile.e2e-tester
|
|
else
|
|
git checkout HEAD -- Dockerfile.e2e-tester
|
|
fi
|
|
|
|
- name: Build and push E2E tester image
|
|
id: e2e_tester_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
PLAYWRIGHT_BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
|
|
|
|
retry_base_pull() {
|
|
image_ref="$1"
|
|
attempts="${2:-6}"
|
|
backoff="${3:-5}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "base pull attempt ${attempt}/${attempts} for ${image_ref}"
|
|
|
|
if getent hosts "${GITEA_REGISTRY_HOST}" >/dev/null 2>&1; then
|
|
echo "${GITEA_REGISTRY_HOST} DNS resolution: ok"
|
|
else
|
|
echo "${GITEA_REGISTRY_HOST} DNS resolution: failed"
|
|
fi
|
|
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true
|
|
|
|
if docker pull "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying base image pull in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e"
|
|
E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}"
|
|
|
|
retry_base_pull "${PLAYWRIGHT_BASE_IMAGE}" 6 5
|
|
DOCKER_BUILDKIT=0 docker build --pull=false -f Dockerfile.e2e-tester \
|
|
--build-arg PLAYWRIGHT_BASE_IMAGE="${PLAYWRIGHT_BASE_IMAGE}" \
|
|
-t plex-playlist-e2e:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}"
|
|
|
|
retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4
|
|
retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3
|
|
|
|
E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve E2E tester digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
runtime_images:
|
|
name: Build and Publish Runtime Images (Legacy Disabled)
|
|
runs-on: ubuntu-act-8gb
|
|
if: false
|
|
needs: [build_cicd, cicd-tests-complete, build-backend-base-image, build-frontend-base-image]
|
|
timeout-minutes: 45
|
|
outputs:
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
|
|
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
|
|
deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
|
|
deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *resolve_head_sha_from_build_step
|
|
|
|
- name: Minimal checkout for runtime image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- \
|
|
.dockerignore \
|
|
Dockerfile.backend \
|
|
Dockerfile.frontend \
|
|
backend \
|
|
frontend \
|
|
scripts/check-dockerfile-boundaries.sh \
|
|
scripts/verify-deployable-image-purity.sh
|
|
else
|
|
git checkout HEAD -- \
|
|
.dockerignore \
|
|
Dockerfile.backend \
|
|
Dockerfile.frontend \
|
|
backend \
|
|
frontend \
|
|
scripts/check-dockerfile-boundaries.sh \
|
|
scripts/verify-deployable-image-purity.sh
|
|
fi
|
|
|
|
- name: Verify deployable runtime boundaries
|
|
run: |
|
|
set -euo pipefail
|
|
bash ./scripts/check-dockerfile-boundaries.sh
|
|
|
|
- name: Build and verify deployable runtime image purity
|
|
env:
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" .
|
|
docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" .
|
|
|
|
bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend
|
|
bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend
|
|
|
|
- name: Push deployable backend runtime image
|
|
id: deployable_backend_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
echo "${op_name} failed. Probing registry endpoint before retry..."
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
|
|
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
|
|
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
|
|
retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
|
|
|
|
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve backend digest after push/pull retries"
|
|
exit 1
|
|
fi
|
|
|
|
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Push deployable frontend runtime image
|
|
id: deployable_frontend_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
echo "${op_name} failed. Probing registry endpoint before retry..."
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
|
|
DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
|
|
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
|
|
|
|
retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
|
|
|
|
DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
|
|
|
|
if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve frontend digest after push/pull retries"
|
|
exit 1
|
|
fi
|
|
|
|
echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-backend-main-image:
|
|
name: Build Backend Main Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, cicd-tests-complete, build-backend-base-image]
|
|
outputs:
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
|
|
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
|
|
steps:
|
|
- *resolve_head_sha_from_build_step
|
|
|
|
- name: Minimal checkout for backend main image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- \
|
|
.dockerignore \
|
|
Dockerfile.backend \
|
|
backend \
|
|
scripts/verify-deployable-image-purity.sh
|
|
else
|
|
git checkout HEAD -- \
|
|
.dockerignore \
|
|
Dockerfile.backend \
|
|
backend \
|
|
scripts/verify-deployable-image-purity.sh
|
|
fi
|
|
|
|
- name: Build and push backend main image
|
|
id: deployable_backend_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" .
|
|
bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
|
|
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
|
|
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
|
|
retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve backend main digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-frontend-main-image:
|
|
name: Build Frontend Main Image
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, cicd-tests-complete, build-frontend-base-image]
|
|
outputs:
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
|
|
deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
|
|
steps:
|
|
- *resolve_head_sha_from_build_step
|
|
|
|
- name: Minimal checkout for frontend main image inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- \
|
|
.dockerignore \
|
|
Dockerfile.frontend \
|
|
frontend \
|
|
scripts/verify-deployable-image-purity.sh
|
|
else
|
|
git checkout HEAD -- \
|
|
.dockerignore \
|
|
Dockerfile.frontend \
|
|
frontend \
|
|
scripts/verify-deployable-image-purity.sh
|
|
fi
|
|
|
|
- name: Build and push frontend main image
|
|
id: deployable_frontend_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" .
|
|
bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend
|
|
|
|
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
|
DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
|
|
DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
|
|
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
|
|
|
|
retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
|
|
|
|
DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve frontend main digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
production-images-complete:
|
|
name: Production Images Complete
|
|
runs-on: ubuntu-act
|
|
needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
|
|
if: always()
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Confirm required production/tester images are complete
|
|
run: |
|
|
set -euo pipefail
|
|
backend_main_status="${{ needs['build-backend-main-image'].result }}"
|
|
frontend_main_status="${{ needs['build-frontend-main-image'].result }}"
|
|
integration_tester_status="${{ needs['build-integration-tester-image'].result }}"
|
|
e2e_tester_status="${{ needs['build-e2e-tester-image'].result }}"
|
|
|
|
echo "build-backend-main-image=${backend_main_status}"
|
|
echo "build-frontend-main-image=${frontend_main_status}"
|
|
echo "build-integration-tester-image=${integration_tester_status}"
|
|
echo "build-e2e-tester-image=${e2e_tester_status}"
|
|
|
|
if [ "${backend_main_status}" != "success" ] || [ "${frontend_main_status}" != "success" ] || [ "${integration_tester_status}" != "success" ] || [ "${e2e_tester_status}" != "success" ]; then
|
|
echo "❌ Production Images Complete gate failed"
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Production, integration tester, and E2E tester images are complete"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
runtime-images-postmortem:
|
|
name: Runtime Images Failure Postmortem
|
|
# Run on the broader runner pool so we can still capture diagnostics when
|
|
# runtime_images fails during platform "Set up job" before any step executes.
|
|
runs-on: ubuntu-act
|
|
needs: [runtime_images, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
|
|
if: always() && (needs.runtime_images.result == 'failure' || needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure')
|
|
timeout-minutes: 10
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- name: Collect runtime_images postmortem context
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
set +e
|
|
|
|
echo "=== Postmortem Context ==="
|
|
echo "workflow=${GITHUB_WORKFLOW:-unknown}"
|
|
echo "run_id=${GITHUB_RUN_ID:-unknown}"
|
|
echo "run_number=${GITHUB_RUN_NUMBER:-unknown}"
|
|
echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
|
|
echo "repository=${GITHUB_REPOSITORY:-unknown}"
|
|
echo "server_url=${GITHUB_SERVER_URL:-unknown}"
|
|
echo "job_result_runtime_images=${{ needs.runtime_images.result }}"
|
|
|
|
echo "=== Local Runner Telemetry (postmortem job host) ==="
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
|
|
echo "=== Registry Reachability Probe ==="
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
if [ -z "${GITHUB_TOKEN:-}" ]; then
|
|
echo "No GITHUB_TOKEN available for API diagnostics"
|
|
exit 0
|
|
fi
|
|
|
|
API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
|
|
JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200"
|
|
|
|
echo "=== Actions API Job Summary ==="
|
|
echo "jobs_url=${JOBS_URL}"
|
|
if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then
|
|
echo "Could not fetch job metadata from Actions API"
|
|
exit 0
|
|
fi
|
|
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 -c "$(printf '%s\n' \
|
|
'import json' \
|
|
'from pathlib import Path' \
|
|
'' \
|
|
'path = Path("/tmp/actions-jobs.json")' \
|
|
'payload = json.loads(path.read_text())' \
|
|
'jobs = payload.get("jobs") or payload.get("data") or []' \
|
|
'' \
|
|
'print("job_count=", len(jobs))' \
|
|
'' \
|
|
'def _g(job, *keys):' \
|
|
' for key in keys:' \
|
|
' if key in job and job[key] not in (None, ""):' \
|
|
' return job[key]' \
|
|
' return "unknown"' \
|
|
'' \
|
|
'for job in jobs:' \
|
|
' name = _g(job, "name", "job_name")' \
|
|
' status = _g(job, "status")' \
|
|
' conclusion = _g(job, "conclusion", "result")' \
|
|
' runner = _g(job, "runner_name", "runner")' \
|
|
' started = _g(job, "started_at", "start_time")' \
|
|
' completed = _g(job, "completed_at", "end_time")' \
|
|
' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \
|
|
'' \
|
|
'target = None' \
|
|
'for job in jobs:' \
|
|
' name = str(_g(job, "name", "job_name"))' \
|
|
' if "Build and Publish Runtime Images" in name:' \
|
|
' target = job' \
|
|
' break' \
|
|
'' \
|
|
'if target is None:' \
|
|
' print("runtime_images_job=not_found")' \
|
|
'else:' \
|
|
' print("=== runtime_images job detail ===")' \
|
|
' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \
|
|
' if key in target:' \
|
|
' print(f"{key}={target[key]}")')"
|
|
else
|
|
echo "python3 unavailable; emitting raw jobs payload head"
|
|
sed -n '1,120p' /tmp/actions-jobs.json || true
|
|
fi
|
|
|
|
exit 0
|
|
|
|
source-lanes-postmortem:
|
|
name: Source Lanes Failure Postmortem
|
|
# Capture follow-up diagnostics when any source lane dies during platform
|
|
# setup before step logging is available.
|
|
runs-on: ubuntu-act
|
|
needs: [backend-tests, precommit-tests, frontend-tests, xdoctest]
|
|
if: always() && (needs['backend-tests'].result == 'failure' || needs['precommit-tests'].result == 'failure' || needs['frontend-tests'].result == 'failure' || needs.xdoctest.result == 'failure')
|
|
timeout-minutes: 10
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- name: Collect source lane postmortem context
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
set +e
|
|
|
|
echo "=== Postmortem Context ==="
|
|
echo "workflow=${GITHUB_WORKFLOW:-unknown}"
|
|
echo "run_id=${GITHUB_RUN_ID:-unknown}"
|
|
echo "run_number=${GITHUB_RUN_NUMBER:-unknown}"
|
|
echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
|
|
echo "repository=${GITHUB_REPOSITORY:-unknown}"
|
|
echo "server_url=${GITHUB_SERVER_URL:-unknown}"
|
|
echo "backend_tests_result=${{ needs['backend-tests'].result }}"
|
|
echo "precommit_tests_result=${{ needs['precommit-tests'].result }}"
|
|
echo "frontend_tests_result=${{ needs['frontend-tests'].result }}"
|
|
echo "xdoctest_result=${{ needs.xdoctest.result }}"
|
|
|
|
echo "=== Local Runner Telemetry (postmortem job host) ==="
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
|
|
echo "=== Registry Reachability Probe ==="
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
if [ -z "${GITHUB_TOKEN:-}" ]; then
|
|
echo "No GITHUB_TOKEN available for API diagnostics"
|
|
exit 0
|
|
fi
|
|
|
|
API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
|
|
JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200"
|
|
|
|
echo "=== Actions API Job Summary ==="
|
|
echo "jobs_url=${JOBS_URL}"
|
|
if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then
|
|
echo "Could not fetch job metadata from Actions API"
|
|
exit 0
|
|
fi
|
|
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 -c "$(printf '%s\n' \
|
|
'import json' \
|
|
'from pathlib import Path' \
|
|
'' \
|
|
'path = Path("/tmp/actions-jobs.json")' \
|
|
'payload = json.loads(path.read_text())' \
|
|
'jobs = payload.get("jobs") or payload.get("data") or []' \
|
|
'' \
|
|
'print("job_count=", len(jobs))' \
|
|
'' \
|
|
'def _g(job, *keys):' \
|
|
' for key in keys:' \
|
|
' if key in job and job[key] not in (None, ""):' \
|
|
' return job[key]' \
|
|
' return "unknown"' \
|
|
'' \
|
|
'targets = {' \
|
|
' "Backend Tests",' \
|
|
' "Pre-commit Checks",' \
|
|
' "Frontend Tests",' \
|
|
' "Backend Doctests",' \
|
|
'}' \
|
|
'' \
|
|
'for job in jobs:' \
|
|
' name = str(_g(job, "name", "job_name"))' \
|
|
' if name in targets:' \
|
|
' status = _g(job, "status")' \
|
|
' conclusion = _g(job, "conclusion", "result")' \
|
|
' runner = _g(job, "runner_name", "runner")' \
|
|
' started = _g(job, "started_at", "start_time")' \
|
|
' completed = _g(job, "completed_at", "end_time")' \
|
|
' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")')"
|
|
else
|
|
echo "python3 unavailable; emitting raw jobs payload head"
|
|
sed -n '1,120p' /tmp/actions-jobs.json || true
|
|
fi
|
|
|
|
exit 0
|
|
|
|
integration-tests:
|
|
name: Runtime Black-Box Integration Tests
|
|
# Use the broader runner pool; only docker build steps are 8gb-exclusive.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 20
|
|
needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image]
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- name: Run runtime black-box integration checks via compose
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }}
|
|
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }}
|
|
INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }}
|
|
run: |
|
|
set -euo pipefail
|
|
set -o pipefail
|
|
|
|
RUN_ID="${GITHUB_RUN_ID:-local}"
|
|
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
|
|
LOG_FILE="$(mktemp)"
|
|
COMPOSE_FILE="/tmp/compose.ci.integration.yaml"
|
|
COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}"
|
|
DB_PASSWORD="plex_password"
|
|
DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist"
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "❌ Docker is required"
|
|
exit 1
|
|
fi
|
|
if ! docker compose version >/dev/null 2>&1; then
|
|
echo "❌ docker compose plugin is required"
|
|
exit 1
|
|
fi
|
|
|
|
cat >"${COMPOSE_FILE}" <<'EOF'
|
|
services:
|
|
database:
|
|
image: postgres:16-alpine
|
|
environment:
|
|
POSTGRES_DB: plex_playlist
|
|
POSTGRES_USER: plex_user
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 12
|
|
|
|
backend:
|
|
image: ${DEPLOYABLE_BACKEND_IMAGE}
|
|
environment:
|
|
DATABASE_URL: ${DB_URL}
|
|
ENVIRONMENT: production
|
|
depends_on:
|
|
database:
|
|
condition: service_healthy
|
|
EOF
|
|
|
|
fetch_backend_endpoint() {
|
|
endpoint_path="$1"
|
|
output_file="$2"
|
|
probe_tmp="$(mktemp)"
|
|
|
|
if ! docker run --rm \
|
|
--network "${COMPOSE_PROJECT_NAME}_default" \
|
|
"${INTEGRATION_TESTER_DIGEST_REF}" \
|
|
python -c "$(printf '%s\n' \
|
|
'import sys' \
|
|
'import urllib.error' \
|
|
'import urllib.request' \
|
|
'endpoint_path = sys.argv[1]' \
|
|
'url = f"http://backend:8000{endpoint_path}"' \
|
|
'try:' \
|
|
' with urllib.request.urlopen(url, timeout=2) as response:' \
|
|
' body = response.read().decode("utf-8", errors="replace")' \
|
|
' print(response.status)' \
|
|
' print(body)' \
|
|
'except urllib.error.HTTPError as err:' \
|
|
' body = err.read().decode("utf-8", errors="replace")' \
|
|
' print(err.code)' \
|
|
' print(body)' \
|
|
'except Exception:' \
|
|
' print("000")' \
|
|
' print("")')" \
|
|
"${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then
|
|
: >"${output_file}"
|
|
echo "000"
|
|
rm -f "${probe_tmp}"
|
|
return 0
|
|
fi
|
|
|
|
http_code="$(head -n 1 "${probe_tmp}")"
|
|
tail -n +2 "${probe_tmp}" >"${output_file}"
|
|
rm -f "${probe_tmp}"
|
|
echo "${http_code}"
|
|
}
|
|
|
|
dump_failure_context() {
|
|
echo "=== Runtime Integration Failure Context ==="
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true
|
|
echo "--- Compose logs (tail 200) ---"
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true
|
|
}
|
|
|
|
cleanup() {
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "pull" ]; then
|
|
if docker pull "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
return 1
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
{
|
|
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Missing deployable backend image references from dispatch inputs"
|
|
exit 1
|
|
fi
|
|
|
|
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
|
|
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
|
|
EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
|
ACTUAL_TAG_DIGEST_REF="$({
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
|
|
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
|
|
| head -n 1
|
|
} || true)"
|
|
|
|
if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then
|
|
echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then
|
|
echo "❌ Tag and digest mismatch"
|
|
echo "expected=${EXPECTED_DIGEST_REF}"
|
|
echo "actual=${ACTUAL_TAG_DIGEST_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}"
|
|
|
|
export DB_PASSWORD DB_URL DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}"
|
|
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend
|
|
|
|
db_ready=false
|
|
for i in $(seq 1 30); do
|
|
if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
|
|
db_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${db_ready}" != "true" ]; then
|
|
echo "❌ Database did not become ready"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
backend_ready=false
|
|
for i in $(seq 1 40); do
|
|
backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
|
|
if [ "${backend_running_count}" -eq 0 ]; then
|
|
echo "❌ Backend container exited before becoming healthy"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
|
if [ "${health_code}" = "200" ]; then
|
|
backend_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${backend_ready}" != "true" ]; then
|
|
echo "❌ Backend did not become healthy"
|
|
cat /tmp/blackbox-health.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)"
|
|
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
|
|
echo "❌ Root endpoint validation failed"
|
|
cat /tmp/blackbox-root.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)"
|
|
if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
|
|
echo "❌ Compatibility endpoint validation failed"
|
|
cat /tmp/blackbox-compatibility.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
|
if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then
|
|
echo "❌ Health endpoint validation failed"
|
|
cat /tmp/blackbox-health.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Runtime integration checks passed"
|
|
} 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Runtime integration checks failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of runtime integration output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
integration-tests-postmortem:
|
|
name: Integration Tests Failure Postmortem
|
|
runs-on: ubuntu-act
|
|
needs: integration-tests
|
|
if: always() && needs['integration-tests'].result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Integration postmortem diagnostics
|
|
run: |
|
|
echo "=== Integration Tests Postmortem ==="
|
|
echo "integration_tests_result=${{ needs['integration-tests'].result }}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
e2e-tests:
|
|
name: End-to-End Tests
|
|
# Use ubuntu-act runner pool for consistent availability.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 45
|
|
needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- name: Run E2E tests against runtime services via compose
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }}
|
|
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }}
|
|
DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }}
|
|
DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_digest_ref }}
|
|
INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }}
|
|
E2E_TESTER_DIGEST_REF: ${{ needs['build-e2e-tester-image'].outputs.e2e_tester_digest_ref }}
|
|
run: |
|
|
set -euo pipefail
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
ARTIFACT_DIR="$(mktemp -d)"
|
|
RUN_ID="${GITHUB_RUN_ID:-local}"
|
|
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
|
|
COMPOSE_FILE="/tmp/compose.ci.e2e.yaml"
|
|
COMPOSE_PROJECT_NAME="plex-e2e-${RUN_ID}-${RUN_ATTEMPT}"
|
|
DB_PASSWORD="plex_password"
|
|
DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist"
|
|
E2E_REPO_DIR="$(mktemp -d)"
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "❌ Docker is required"
|
|
exit 1
|
|
fi
|
|
if ! docker compose version >/dev/null 2>&1; then
|
|
echo "❌ docker compose plugin is required"
|
|
exit 1
|
|
fi
|
|
|
|
cat >"${COMPOSE_FILE}" <<'EOF'
|
|
services:
|
|
database:
|
|
image: postgres:16-alpine
|
|
environment:
|
|
POSTGRES_DB: plex_playlist
|
|
POSTGRES_USER: plex_user
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 12
|
|
|
|
backend:
|
|
image: ${DEPLOYABLE_BACKEND_IMAGE}
|
|
environment:
|
|
DATABASE_URL: ${DB_URL}
|
|
ENVIRONMENT: production
|
|
depends_on:
|
|
database:
|
|
condition: service_healthy
|
|
|
|
frontend:
|
|
image: ${DEPLOYABLE_FRONTEND_IMAGE}
|
|
depends_on:
|
|
backend:
|
|
condition: service_started
|
|
EOF
|
|
|
|
fetch_http() {
|
|
container_name="$1"
|
|
endpoint_path="$2"
|
|
output_file="$3"
|
|
probe_tmp="$(mktemp)"
|
|
|
|
run_probe_with_python() {
|
|
python_bin="$1"
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" "${python_bin}" -c "$(printf '%s\n' \
|
|
'import sys' \
|
|
'import urllib.error' \
|
|
'import urllib.request' \
|
|
'endpoint_path = sys.argv[1]' \
|
|
'url = f"http://127.0.0.1:8000{endpoint_path}"' \
|
|
'try:' \
|
|
' with urllib.request.urlopen(url, timeout=2) as response:' \
|
|
' body = response.read().decode("utf-8", errors="replace")' \
|
|
' print(response.status)' \
|
|
' print(body)' \
|
|
'except urllib.error.HTTPError as err:' \
|
|
' body = err.read().decode("utf-8", errors="replace")' \
|
|
' print(err.code)' \
|
|
' print(body)' \
|
|
'except Exception:' \
|
|
' print("000")' \
|
|
' print("")')" "${endpoint_path}"
|
|
}
|
|
|
|
probe_err="$(mktemp)"
|
|
|
|
if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then
|
|
:
|
|
else
|
|
: >"${output_file}"
|
|
{
|
|
echo '{"status":"probe_exec_failed"}'
|
|
sed -n '1,5p' "${probe_err}" || true
|
|
} >"${output_file}"
|
|
rm -f "${probe_tmp}"
|
|
rm -f "${probe_err}"
|
|
echo "000"
|
|
return 0
|
|
fi
|
|
|
|
http_code="$(head -n 1 "${probe_tmp}")"
|
|
tail -n +2 "${probe_tmp}" >"${output_file}"
|
|
rm -f "${probe_tmp}"
|
|
rm -f "${probe_err}"
|
|
echo "${http_code}"
|
|
}
|
|
|
|
dump_failure_context() {
|
|
echo "=== Runtime E2E Failure Context ==="
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true
|
|
echo "--- Compose logs (tail 200) ---"
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true
|
|
echo "--- Artifact directory ---"
|
|
find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
|
|
}
|
|
|
|
cleanup() {
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
|
|
rm -rf "${E2E_REPO_DIR}" >/dev/null 2>&1 || true
|
|
rm -f ~/.ssh/id_rsa >/dev/null 2>&1 || true
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "pull" ]; then
|
|
if docker pull "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
return 1
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Missing deployable runtime image references from dispatch inputs"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" "${E2E_REPO_DIR}"
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git -C "${E2E_REPO_DIR}" fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git -C "${E2E_REPO_DIR}" checkout FETCH_HEAD -- frontend
|
|
else
|
|
git -C "${E2E_REPO_DIR}" checkout HEAD -- frontend
|
|
fi
|
|
|
|
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
|
|
EXPECTED_BACKEND_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
|
ACTUAL_BACKEND_TAG_DIGEST_REF="$({
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
|
|
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
|
|
| head -n 1
|
|
} || true)"
|
|
|
|
if [ -z "${ACTUAL_BACKEND_TAG_DIGEST_REF}" ]; then
|
|
echo "❌ Could not resolve backend digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${ACTUAL_BACKEND_TAG_DIGEST_REF}" != "${EXPECTED_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Backend tag and digest mismatch"
|
|
echo "expected=${EXPECTED_BACKEND_DIGEST_REF}"
|
|
echo "actual=${ACTUAL_BACKEND_TAG_DIGEST_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
DEPLOYABLE_FRONTEND_REPO="${DEPLOYABLE_FRONTEND_DIGEST_REF%%@*}"
|
|
EXPECTED_FRONTEND_DIGEST_REF="${DEPLOYABLE_FRONTEND_DIGEST_REF}"
|
|
ACTUAL_FRONTEND_TAG_DIGEST_REF="$({
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" \
|
|
| grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" \
|
|
| head -n 1
|
|
} || true)"
|
|
|
|
if [ -z "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" ]; then
|
|
echo "❌ Could not resolve frontend digest from tag reference: ${DEPLOYABLE_FRONTEND_TAG_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" != "${EXPECTED_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Frontend tag and digest mismatch"
|
|
echo "expected=${EXPECTED_FRONTEND_DIGEST_REF}"
|
|
echo "actual=${ACTUAL_FRONTEND_TAG_DIGEST_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}"
|
|
echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}"
|
|
|
|
export DB_PASSWORD DB_URL E2E_ARTIFACT_DIR="${ARTIFACT_DIR}"
|
|
export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}"
|
|
export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}"
|
|
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend
|
|
|
|
db_ready=false
|
|
for i in $(seq 1 30); do
|
|
if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
|
|
db_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${db_ready}" != "true" ]; then
|
|
echo "❌ Database did not become ready"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
backend_ready=false
|
|
for i in $(seq 1 40); do
|
|
backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
|
|
if [ "${backend_running_count}" -eq 0 ]; then
|
|
echo "❌ Backend container exited before becoming healthy"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
health_code="$(fetch_http "backend" "/health" /tmp/e2e-backend-health.json)"
|
|
if [ "${health_code}" = "200" ]; then
|
|
backend_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${backend_ready}" != "true" ]; then
|
|
echo "❌ Backend did not become healthy"
|
|
cat /tmp/e2e-backend-health.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
root_code="$(fetch_http "backend" "/" /tmp/e2e-backend-root.json)"
|
|
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/e2e-backend-root.json; then
|
|
echo "❌ Backend root endpoint validation failed"
|
|
cat /tmp/e2e-backend-root.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
frontend_ready=false
|
|
for i in $(seq 1 30); do
|
|
frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)"
|
|
if [ "${frontend_running_count}" -gt 0 ]; then
|
|
if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T frontend sh -lc 'wget -q -T 2 -O /dev/null http://127.0.0.1/'; then
|
|
frontend_ready=true
|
|
break
|
|
fi
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${frontend_ready}" != "true" ]; then
|
|
echo "❌ Frontend did not become reachable"
|
|
cat /tmp/e2e-frontend-root.html 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
docker run --rm \
|
|
--network "${COMPOSE_PROJECT_NAME}_default" \
|
|
-e CI=true \
|
|
-e PLAYWRIGHT_BASE_URL="http://frontend:80" \
|
|
-e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \
|
|
-e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \
|
|
-v "${E2E_REPO_DIR}/frontend:/workspace/frontend" \
|
|
-v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \
|
|
"${E2E_TESTER_DIGEST_REF}" \
|
|
bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \
|
|
2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ E2E tests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of E2E output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
echo "--- E2E artifact files ---"
|
|
find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
|
|
dump_failure_context
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
echo "=== E2E artifact files ==="
|
|
find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
e2e-tests-postmortem:
|
|
name: E2E Tests Failure Postmortem
|
|
runs-on: ubuntu-act
|
|
needs: e2e-tests
|
|
if: always() && needs['e2e-tests'].result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: E2E postmortem diagnostics
|
|
run: |
|
|
echo "=== E2E Tests Postmortem ==="
|
|
echo "e2e_tests_result=${{ needs['e2e-tests'].result }}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" || true
|