a6180a187d
Some checks failed
CICD / Build and Push CICD Images (push) Failing after 11m46s
CICD / Source Checks (push) Has been skipped
CICD / Dependency Audits (Informational) (push) Has been skipped
CICD / Build Release Images (push) Has been skipped
CICD / Build Tester Images (push) Has been skipped
CICD / Build CICD Image Failure Postmortem (push) Successful in 18s
CICD / CICD Tests Complete (push) Failing after 55s
CICD / Production Images Complete (push) Failing after 57s
CICD / Runtime Black-Box Integration Tests (push) Has been skipped
CICD / End-to-End Tests (push) Has been skipped
CICD / Production Image Failures Postmortem (push) Has been skipped
CICD / Integration Tests Failure Postmortem (push) Has been skipped
CICD / Source Lanes Failure Postmortem (push) Has been skipped
CICD / Promote Staging Images To Release (push) Has been skipped
CICD / E2E Tests Failure Postmortem (push) Has been skipped
2830 lines
117 KiB
YAML
2830 lines
117 KiB
YAML
name: CICD
|
|
|
|
on:
|
|
push:
|
|
branches: [main, develop]
|
|
pull_request:
|
|
branches: [main, develop]
|
|
workflow_dispatch:
|
|
inputs:
|
|
head_sha:
|
|
description: Commit SHA to process
|
|
required: false
|
|
force_rebuild_base:
|
|
description: Force CICD base rebuild
|
|
required: false
|
|
default: "false"
|
|
|
|
env:
|
|
GITEA_SSH_HOST: kankali.darkhelm.lan
|
|
GITEA_SSH_PORT: "2222"
|
|
GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
|
|
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
|
GITEA_REGISTRY_IP: 10.18.75.2
|
|
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
|
GITEA_REGISTRY_AUTH_HOST: kankali.darkhelm.lan
|
|
GITEA_REGISTRY_AUTH_FALLBACK_HOST: dogar.darkhelm.org
|
|
CI_PRUNE_AT_JOB_START: "true"
|
|
CI_PRUNE_THRESHOLD_PERCENT: "90"
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
concurrency:
|
|
group: cicd-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
build_cicd:
|
|
name: Build and Push CICD Images
|
|
runs-on: ubuntu-act-8gb
|
|
timeout-minutes: 35
|
|
outputs:
|
|
base_hash: ${{ steps.base-state.outputs.base_hash }}
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
steps:
|
|
- &identify_runner_step
|
|
name: Identify runner
|
|
run: |
|
|
echo "=== Runner Identity ==="
|
|
echo "runner_name=${RUNNER_NAME:-}"
|
|
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
|
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
|
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
|
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
|
echo "runner_os=${RUNNER_OS:-unknown}"
|
|
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
|
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
- &pre_job_prune_step
|
|
name: Pre-job Docker prune
|
|
run: |
|
|
if [ "${CI_PRUNE_AT_JOB_START:-false}" != "true" ] || ! command -v docker >/dev/null 2>&1; then
|
|
echo "Skipping pre-job prune"
|
|
exit 0
|
|
fi
|
|
|
|
used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
|
|
echo "disk_used_before=${used_before}%"
|
|
echo "prune_threshold=${threshold}%"
|
|
|
|
if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
|
|
docker builder prune -af || true
|
|
docker system prune -af --volumes || true
|
|
used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
echo "disk_used_after=${used_after}%"
|
|
else
|
|
echo "Skipping prune; usage below threshold"
|
|
fi
|
|
|
|
- name: Resolve head SHA
|
|
id: meta
|
|
env:
|
|
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
|
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
|
run: |
|
|
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
|
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Minimal checkout for base inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore versions/ci.env backend/pyproject.toml frontend/package.json scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
|
|
else
|
|
git checkout HEAD -- Dockerfile.cicd-base .dockerignore versions/ci.env backend/pyproject.toml frontend/package.json scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
|
|
fi
|
|
|
|
chmod +x scripts/compute-cicd-base-hash.sh
|
|
chmod +x scripts/resolve-mirrored-image.sh
|
|
|
|
- name: Compute base hash and inspect registry
|
|
id: base-state
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
FORCE_REBUILD: ${{ github.event.inputs.force_rebuild_base }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
|
|
echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
|
|
fi
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
ensure_registry_auth_realm_host
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
|
|
BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
|
|
BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest"
|
|
|
|
echo "base_hash=${BASE_HASH}" >> "$GITHUB_OUTPUT"
|
|
echo "base_ref_hash=${BASE_REF_HASH}" >> "$GITHUB_OUTPUT"
|
|
echo "base_ref_latest=${BASE_REF_LATEST}" >> "$GITHUB_OUTPUT"
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
ensure_registry_auth_realm_host
|
|
|
|
if ! docker_login_with_retry 5 3 >/dev/null; then
|
|
echo "❌ Failed docker login after retries"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${FORCE_REBUILD:-false}" = "true" ]; then
|
|
echo "needs_build=true" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
|
|
if timeout 900 docker pull "${BASE_REF_HASH}" >/dev/null 2>&1; then
|
|
echo "needs_build=false" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "needs_build=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Build and push base image
|
|
if: steps.base-state.outputs.needs_build == 'true'
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
BASE_HASH: ${{ steps.base-state.outputs.base_hash }}
|
|
BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }}
|
|
BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
|
|
echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
|
|
fi
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
ensure_registry_auth_realm_host
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
docker_login_with_retry 5 3
|
|
|
|
set -a
|
|
# shellcheck source=versions/ci.env
|
|
source versions/ci.env
|
|
set +a
|
|
|
|
PYTHON_VERSION="$(python3 -c "from pathlib import Path; import re, tomllib, sys; pyproject = tomllib.loads(Path('backend/pyproject.toml').read_text()); requires_python = pyproject['project']['requires-python']; match = re.search(r'([0-9]+[.][0-9]+)', requires_python); sys.exit('could not derive backend Python version') if not match else None; print(match.group(1))")"
|
|
|
|
NODE_MAJOR="$(node -p "const pkg = require('./frontend/package.json'); const match = String(pkg.engines?.node ?? '').match(/(\\d+)/); if (!match) throw new Error('could not derive frontend Node major'); match[1]")"
|
|
|
|
PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/${PLAYWRIGHT_MIRROR_IMAGE}"
|
|
PLAYWRIGHT_BROWSERS_UPSTREAM_TAG="${PLAYWRIGHT_IMAGE}"
|
|
PLAYWRIGHT_BROWSERS_LOCAL_TAG="$(./scripts/resolve-mirrored-image.sh \
|
|
--image "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" \
|
|
--mirror-image "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" \
|
|
--upstream-image "${PLAYWRIGHT_BROWSERS_UPSTREAM_TAG}" \
|
|
--registry-user "${REGISTRY_USER}" \
|
|
--registry-password-env PACKAGE_ACCESS_TOKEN)"
|
|
|
|
PLAYWRIGHT_BROWSERS_IMAGE="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${PLAYWRIGHT_BROWSERS_LOCAL_TAG}" | head -n 1; } || true)"
|
|
if [ -z "${PLAYWRIGHT_BROWSERS_IMAGE}" ]; then
|
|
echo "❌ Unable to resolve Playwright browsers digest reference"
|
|
exit 1
|
|
fi
|
|
|
|
docker build --progress=plain -f Dockerfile.cicd-base \
|
|
--build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \
|
|
--build-arg BASE_IMAGE_HASH="${BASE_HASH}" \
|
|
--build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
|
|
--build-arg NODE_MAJOR="${NODE_MAJOR}" \
|
|
--build-arg PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION}" \
|
|
--build-arg PLAYWRIGHT_DISTRO="${PLAYWRIGHT_DISTRO}" \
|
|
--build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \
|
|
-t cicd-base:latest .
|
|
|
|
docker tag cicd-base:latest "${BASE_REF_HASH}"
|
|
docker tag cicd-base:latest "${BASE_REF_LATEST}"
|
|
retry_registry_op push "${BASE_REF_HASH}" 5 4
|
|
retry_registry_op push "${BASE_REF_LATEST}" 5 4
|
|
|
|
- name: Minimal checkout for CICD build inputs
|
|
env:
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
|
|
|
|
if [ ! -d .git ]; then
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
|
fi
|
|
|
|
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
|
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
|
git checkout FETCH_HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base versions/ci.env scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
|
|
else
|
|
git checkout HEAD -- .dockerignore Dockerfile.cicd Dockerfile.cicd-base versions/ci.env scripts/compute-cicd-base-hash.sh scripts/resolve-mirrored-image.sh
|
|
fi
|
|
chmod +x scripts/compute-cicd-base-hash.sh
|
|
chmod +x scripts/resolve-mirrored-image.sh
|
|
|
|
- name: Build and push complete CICD image
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
BASE_HASH_FROM_BUILD: ${{ steps.base-state.outputs.base_hash }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
trap 'rm -f /tmp/ssh_key' EXIT
|
|
|
|
trim_spaces() {
|
|
printf '%s' "$1" | tr -d '[:space:]'
|
|
}
|
|
|
|
is_hex() {
|
|
value="$1"
|
|
if [ -z "${value}" ]; then
|
|
return 1
|
|
fi
|
|
case "${value}" in
|
|
(*[!0-9a-fA-F]*) return 1 ;;
|
|
(*) return 0 ;;
|
|
esac
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
|
|
echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
|
|
fi
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
ensure_registry_auth_realm_host
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
echo "=== Pre-build disk telemetry ==="
|
|
df -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker system df || true
|
|
used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
|
|
echo "disk_used_before=${used_before}%"
|
|
echo "prune_threshold=${threshold}%"
|
|
if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
|
|
docker builder prune -af || true
|
|
docker system prune -af --volumes || true
|
|
else
|
|
echo "Skipping prune; usage below threshold"
|
|
fi
|
|
echo "=== Post-prune disk telemetry ==="
|
|
df -h || true
|
|
docker system df || true
|
|
fi
|
|
|
|
ensure_registry_auth_realm_host
|
|
|
|
docker_login_with_retry 5 3
|
|
|
|
RESOLVED_HEAD_SHA="$(trim_spaces "${HEAD_SHA:-}")"
|
|
if ! is_hex "${RESOLVED_HEAD_SHA}"; then
|
|
echo "❌ Invalid or empty HEAD_SHA resolved for build_cicd: '${HEAD_SHA:-}'"
|
|
exit 1
|
|
fi
|
|
|
|
BASE_HASH="$(trim_spaces "${BASE_HASH_FROM_BUILD}")"
|
|
if ! is_hex "${BASE_HASH}"; then
|
|
BASE_HASH="$(./scripts/compute-cicd-base-hash.sh | tr -d '[:space:]')"
|
|
fi
|
|
|
|
if ! is_hex "${BASE_HASH}"; then
|
|
echo "❌ Invalid or empty BASE_HASH resolved for build_cicd: '${BASE_HASH_FROM_BUILD}'"
|
|
exit 1
|
|
fi
|
|
|
|
echo "resolved_head_sha=${RESOLVED_HEAD_SHA}"
|
|
echo "resolved_base_hash=${BASE_HASH}"
|
|
|
|
BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
|
|
CICD_LATEST_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest"
|
|
CICD_SHA_REF="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${RESOLVED_HEAD_SHA}"
|
|
|
|
retry_registry_op pull "${BASE_IMAGE}" 5 3
|
|
retry_registry_op pull "${CICD_LATEST_REF}" 5 3 || true
|
|
|
|
echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
|
|
chmod 600 /tmp/ssh_key
|
|
|
|
DOCKER_BUILDKIT=1 docker build -f Dockerfile.cicd \
|
|
--cache-from "${CICD_LATEST_REF}" \
|
|
--secret id=ssh_private_key,src=/tmp/ssh_key \
|
|
--add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
--build-arg GITHUB_SHA="${RESOLVED_HEAD_SHA}" \
|
|
--build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \
|
|
-t cicd:latest .
|
|
|
|
docker tag cicd:latest "${CICD_LATEST_REF}"
|
|
docker tag cicd:latest "${CICD_SHA_REF}"
|
|
retry_registry_op push "${CICD_LATEST_REF}" 5 4
|
|
retry_registry_op push "${CICD_SHA_REF}" 5 4
|
|
|
|
- &failure_diagnostics_step
|
|
name: Failure diagnostics
|
|
if: failure()
|
|
run: |
|
|
echo "=== Failure Diagnostics ==="
|
|
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
|
echo "runner_name=${RUNNER_NAME:-unknown}"
|
|
echo "runner_hostname=${HOSTNAME:-unknown}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
ps aux --sort=-%mem | head -n 30 || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
echo "=== Docker Diagnostics ==="
|
|
docker version || true
|
|
docker info || true
|
|
docker ps -a || true
|
|
docker images --digests | head -n 50 || true
|
|
fi
|
|
dmesg | tail -n 120 || true
|
|
|
|
build-cicd-postmortem:
|
|
name: Build CICD Image Failure Postmortem
|
|
runs-on: ubuntu-act
|
|
needs: build_cicd
|
|
if: always() && needs.build_cicd.result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- name: Build CICD postmortem diagnostics
|
|
run: |
|
|
echo "=== Build CICD Postmortem ==="
|
|
echo "build_cicd_result=${{ needs.build_cicd.result }}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
source-checks:
|
|
name: Source Checks
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 75
|
|
needs: build_cicd
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- &configure_registry_host_step
|
|
name: Configure registry host resolution
|
|
run: |
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then
|
|
echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability."
|
|
fi
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
if [ "${CI_PRUNE_AT_JOB_START:-false}" = "true" ] && command -v docker >/dev/null 2>&1; then
|
|
used_before="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
threshold="${CI_PRUNE_THRESHOLD_PERCENT:-90}"
|
|
echo "disk_used_before=${used_before}%"
|
|
echo "prune_threshold=${threshold}%"
|
|
if [ -n "${used_before}" ] && [ "${used_before}" -ge "${threshold}" ]; then
|
|
docker builder prune -af || true
|
|
docker system prune -af --volumes || true
|
|
used_after="$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
|
|
echo "disk_used_after=${used_after}%"
|
|
else
|
|
echo "Skipping prune; usage below threshold"
|
|
fi
|
|
fi
|
|
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
ensure_registry_auth_realm_host
|
|
|
|
- &ensure_cicd_image_step
|
|
name: Ensure CICD image is available
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || github.sha }}
|
|
run: |
|
|
IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
|
if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
|
|
echo "Using cached CICD image: ${IMAGE}"
|
|
else
|
|
login_ok=false
|
|
for i in 1 2 3 4 5; do
|
|
if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
|
|
login_ok=true
|
|
break
|
|
fi
|
|
if [ "${i}" -lt 5 ]; then
|
|
sleep_seconds=$((3 * i))
|
|
echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
done
|
|
|
|
if [ "${login_ok}" != "true" ]; then
|
|
echo "❌ Failed docker login after retries"
|
|
exit 1
|
|
fi
|
|
|
|
pulled=false
|
|
for i in 1 2 3; do
|
|
echo "Pull attempt ${i}/3 for ${IMAGE}"
|
|
if docker pull "${IMAGE}"; then
|
|
pulled=true
|
|
break
|
|
fi
|
|
if [ "${i}" -lt 3 ]; then
|
|
sleep_seconds=$((5 * i))
|
|
echo "Pull failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
done
|
|
|
|
if [ "${pulled}" != "true" ]; then
|
|
echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
- name: Run backend tests with coverage
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace/backend &&
|
|
source .venv/bin/activate &&
|
|
uv run pytest -v --tb=short -m 'not integration' --cov=src --cov-report=term-missing --cov-fail-under=95
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Backend tests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of backend test output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- name: Run pre-commit checks
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace &&
|
|
if [ ! -d .git ]; then
|
|
git init /workspace
|
|
fi &&
|
|
/workspace/backend/.venv/bin/pre-commit run --all-files --show-diff-on-failure --config .pre-commit-config.yaml
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Pre-commit checks failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of pre-commit output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- name: Run frontend tests with coverage
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace/frontend &&
|
|
yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Frontend tests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of frontend test output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- name: Run backend doctests
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace/backend &&
|
|
.venv/bin/python -m xdoctest src/ --quiet
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Backend doctests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of doctest output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
dependency-audits:
|
|
name: Dependency Audits (Informational)
|
|
runs-on: ubuntu-act
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
needs: build_cicd
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- *ensure_cicd_image_step
|
|
- name: Run frontend dependency audit
|
|
continue-on-error: true
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace &&
|
|
uv --directory backend run poe audit-frontend
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "⚠️ Frontend dependency audit failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of frontend audit output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
echo "Proceeding: frontend audit is informational-only."
|
|
fi
|
|
|
|
- name: Run backend dependency audit
|
|
if: always()
|
|
continue-on-error: true
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
run: |
|
|
set -o pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
|
|
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
|
cd /workspace &&
|
|
uv --directory backend run poe audit-backend
|
|
" 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "⚠️ Backend dependency audit failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of backend audit output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
echo "Proceeding: backend audit is informational-only."
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
cicd-tests-complete:
|
|
name: CICD Tests Complete
|
|
runs-on: ubuntu-act
|
|
needs: [source-checks, dependency-audits]
|
|
if: always()
|
|
steps:
|
|
- name: Confirm all source lanes passed
|
|
run: |
|
|
set -euo pipefail
|
|
source_checks_status="${{ needs['source-checks'].result }}"
|
|
dependency_audits_status="${{ needs['dependency-audits'].result }}"
|
|
|
|
echo "source-checks=${source_checks_status}"
|
|
echo "dependency-audits=${dependency_audits_status}"
|
|
|
|
if [ "${source_checks_status}" != "success" ]; then
|
|
echo "❌ One or more CICD source lanes failed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${dependency_audits_status}" != "success" ]; then
|
|
echo "⚠️ Dependency audit lane reported non-success (informational only)"
|
|
fi
|
|
|
|
echo "✅ Source checks complete"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-release-images:
|
|
name: Build Release Images
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, source-checks]
|
|
outputs:
|
|
head_sha: ${{ steps.meta.outputs.head_sha }}
|
|
backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }}
|
|
backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }}
|
|
frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }}
|
|
frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }}
|
|
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
|
|
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
|
|
deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
|
|
deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
|
|
steps:
|
|
- &resolve_head_sha_from_build_step
|
|
name: Resolve head SHA
|
|
id: meta
|
|
run: |
|
|
echo "head_sha=${{ needs.build_cicd.outputs.head_sha }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *configure_registry_host_step
|
|
- *ensure_cicd_image_step
|
|
- &hydrate_build_context_from_cicd_step
|
|
name: Populate build context from CICD image
|
|
env:
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
|
SOURCE_CONTAINER="$(docker create "${SOURCE_IMAGE}")"
|
|
cleanup() {
|
|
docker rm -f "${SOURCE_CONTAINER}" >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
docker cp "${SOURCE_CONTAINER}:/workspace/." .
|
|
|
|
- name: Build and push backend base image
|
|
id: backend_base_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker_login_with_retry 5 3
|
|
BACKEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-base"
|
|
BACKEND_BASE_TAG_REF="${BACKEND_BASE_REPO}:${HEAD_SHA}"
|
|
BACKEND_BASE_CACHE_REF="${BACKEND_BASE_REPO}:cache"
|
|
|
|
retry_registry_op pull "${BACKEND_BASE_CACHE_REF}" 5 3 || true
|
|
DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
|
|
--target dependencies \
|
|
--cache-from "${BACKEND_BASE_CACHE_REF}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
-t plex-playlist-backend-base:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_TAG_REF}"
|
|
docker tag plex-playlist-backend-base:"${HEAD_SHA}" "${BACKEND_BASE_CACHE_REF}"
|
|
retry_registry_op push "${BACKEND_BASE_TAG_REF}" 5 4
|
|
retry_registry_op push "${BACKEND_BASE_CACHE_REF}" 5 4
|
|
retry_registry_op pull "${BACKEND_BASE_TAG_REF}" 5 3
|
|
|
|
BACKEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${BACKEND_BASE_TAG_REF}" | grep "^${BACKEND_BASE_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${BACKEND_BASE_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve backend base image digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build and push frontend base image
|
|
id: frontend_base_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
retry_cmd "${1:-5}" "${2:-3}" sh -c 'echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin'
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker_login_with_retry 5 3
|
|
FRONTEND_BASE_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-base"
|
|
FRONTEND_BASE_TAG_REF="${FRONTEND_BASE_REPO}:${HEAD_SHA}"
|
|
FRONTEND_BASE_CACHE_REF="${FRONTEND_BASE_REPO}:cache"
|
|
|
|
NODE_ALPINE_TAG="$(node -p "const pkg = require('./frontend/package.json'); const match = String(pkg.engines?.node ?? '').match(/(\\d+)/); if (!match) throw new Error('could not derive frontend Node major'); match[1] + '-alpine'")"
|
|
YARN_VERSION="$(node -p "const pkg = require('./frontend/package.json'); const match = String(pkg.packageManager ?? '').match(/^yarn@(.*)$/); if (!match) throw new Error('could not derive frontend Yarn version'); match[1]")"
|
|
|
|
retry_registry_op pull "${FRONTEND_BASE_CACHE_REF}" 5 3 || true
|
|
BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
|
|
--target deps \
|
|
--cache-from "${FRONTEND_BASE_CACHE_REF}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
--build-arg NODE_ALPINE_TAG="${FRONTEND_NODE_ALPINE_TAG}" \
|
|
--build-arg YARN_VERSION="${FRONTEND_YARN_VERSION}" \
|
|
-t plex-playlist-frontend-base:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_TAG_REF}"
|
|
docker tag plex-playlist-frontend-base:"${HEAD_SHA}" "${FRONTEND_BASE_CACHE_REF}"
|
|
retry_registry_op push "${FRONTEND_BASE_TAG_REF}" 5 4
|
|
retry_registry_op push "${FRONTEND_BASE_CACHE_REF}" 5 4
|
|
retry_registry_op pull "${FRONTEND_BASE_TAG_REF}" 5 3
|
|
|
|
FRONTEND_BASE_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${FRONTEND_BASE_TAG_REF}" | grep "^${FRONTEND_BASE_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${FRONTEND_BASE_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve frontend base image digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build and push backend main image
|
|
id: deployable_backend_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-staging"
|
|
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
|
|
DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache"
|
|
|
|
docker_login_with_retry 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true
|
|
DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
|
|
--cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
-t plex-playlist-backend:"${HEAD_SHA}" .
|
|
bash ./scripts/verify-deployable-image-purity.sh --image plex-playlist-backend:"${HEAD_SHA}" --profile backend
|
|
|
|
docker tag "plex-playlist-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
docker tag "plex-playlist-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}"
|
|
|
|
retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
|
|
retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve backend main digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build and push frontend main image
|
|
id: deployable_frontend_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
ensure_registry_auth_realm_host
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-staging"
|
|
DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
|
|
DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache"
|
|
|
|
NODE_ALPINE_TAG="$(node -p "const pkg = require('./frontend/package.json'); const match = String(pkg.engines?.node ?? '').match(/(\\d+)/); if (!match) throw new Error('could not derive frontend Node major'); match[1] + '-alpine'")"
|
|
YARN_VERSION="$(node -p "const pkg = require('./frontend/package.json'); const match = String(pkg.packageManager ?? '').match(/^yarn@(.*)$/); if (!match) throw new Error('could not derive frontend Yarn version'); match[1]")"
|
|
|
|
docker_login_with_retry 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true
|
|
BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
|
|
--target production \
|
|
--cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
--build-arg NODE_ALPINE_TAG="${FRONTEND_NODE_ALPINE_TAG}" \
|
|
--build-arg YARN_VERSION="${FRONTEND_YARN_VERSION}" \
|
|
-t plex-playlist-frontend:"${HEAD_SHA}" .
|
|
bash ./scripts/verify-deployable-image-purity.sh --image plex-playlist-frontend:"${HEAD_SHA}" --profile frontend
|
|
|
|
docker tag "plex-playlist-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
|
|
docker tag "plex-playlist-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}"
|
|
|
|
retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
|
|
retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
|
|
|
|
DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve frontend main digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
build-tester-images:
|
|
name: Build Tester Images
|
|
runs-on: ubuntu-act-8gb
|
|
needs: [build_cicd, source-checks]
|
|
outputs:
|
|
integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }}
|
|
integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }}
|
|
e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }}
|
|
e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }}
|
|
steps:
|
|
- *resolve_head_sha_from_build_step
|
|
- *configure_registry_host_step
|
|
- *ensure_cicd_image_step
|
|
- *hydrate_build_context_from_cicd_step
|
|
|
|
- name: Build and push integration tester image
|
|
id: integration_tester_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
ensure_registry_auth_realm_host() {
|
|
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
|
fi
|
|
|
|
headers_file="$(mktemp)"
|
|
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
|
|
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
|
|
if [ -n "${realm_url}" ]; then
|
|
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
|
|
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
|
|
if ! grep -q "${realm_host}" /etc/hosts; then
|
|
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
|
|
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "${headers_file}"
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
ensure_registry_auth_realm_host
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker_login_with_retry 5 3
|
|
INTEGRATION_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-integration"
|
|
INTEGRATION_TESTER_TAG_REF="${INTEGRATION_TESTER_REPO}:${HEAD_SHA}"
|
|
INTEGRATION_TESTER_CACHE_REF="${INTEGRATION_TESTER_REPO}:cache"
|
|
|
|
retry_registry_op pull "${INTEGRATION_TESTER_CACHE_REF}" 5 3 || true
|
|
DOCKER_BUILDKIT=1 docker build -f Dockerfile.integration-tester \
|
|
--cache-from "${INTEGRATION_TESTER_CACHE_REF}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
-t plex-playlist-integration:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_TAG_REF}"
|
|
docker tag plex-playlist-integration:"${HEAD_SHA}" "${INTEGRATION_TESTER_CACHE_REF}"
|
|
|
|
retry_registry_op push "${INTEGRATION_TESTER_TAG_REF}" 5 4
|
|
retry_registry_op push "${INTEGRATION_TESTER_CACHE_REF}" 5 4
|
|
retry_registry_op pull "${INTEGRATION_TESTER_TAG_REF}" 5 3
|
|
|
|
INTEGRATION_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${INTEGRATION_TESTER_TAG_REF}" | grep "^${INTEGRATION_TESTER_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${INTEGRATION_TESTER_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve integration tester digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build and push E2E tester image
|
|
id: e2e_tester_ref
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
set -a
|
|
# shellcheck source=versions/ci.env
|
|
source versions/ci.env
|
|
set +a
|
|
|
|
PLAYWRIGHT_BASE_IMAGE_MIRROR="${GITEA_REGISTRY}/${PLAYWRIGHT_MIRROR_IMAGE}"
|
|
PLAYWRIGHT_BASE_IMAGE_UPSTREAM="${PLAYWRIGHT_IMAGE}"
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
retry_base_pull() {
|
|
image_ref="$1"
|
|
attempts="${2:-6}"
|
|
backoff="${3:-5}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "base pull attempt ${attempt}/${attempts} for ${image_ref}"
|
|
|
|
if getent hosts "${GITEA_REGISTRY_HOST}" >/dev/null 2>&1; then
|
|
echo "${GITEA_REGISTRY_HOST} DNS resolution: ok"
|
|
else
|
|
echo "${GITEA_REGISTRY_HOST} DNS resolution: failed"
|
|
fi
|
|
|
|
timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" >/dev/null || true
|
|
|
|
if docker pull "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying base image pull in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker_login_with_retry 5 3
|
|
E2E_TESTER_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-e2e"
|
|
E2E_TESTER_TAG_REF="${E2E_TESTER_REPO}:${HEAD_SHA}"
|
|
E2E_TESTER_CACHE_REF="${E2E_TESTER_REPO}:cache"
|
|
|
|
PLAYWRIGHT_BASE_IMAGE="$(./scripts/resolve-mirrored-image.sh \
|
|
--image "${PLAYWRIGHT_BASE_IMAGE_MIRROR}" \
|
|
--mirror-image "${PLAYWRIGHT_BASE_IMAGE_MIRROR}" \
|
|
--upstream-image "${PLAYWRIGHT_BASE_IMAGE_UPSTREAM}" \
|
|
--registry-user "${REGISTRY_USER}" \
|
|
--registry-password-env PACKAGE_ACCESS_TOKEN)"
|
|
|
|
retry_base_pull "${PLAYWRIGHT_BASE_IMAGE}" 6 5
|
|
retry_registry_op pull "${E2E_TESTER_CACHE_REF}" 5 3 || true
|
|
DOCKER_BUILDKIT=1 docker build --pull=false -f Dockerfile.e2e-tester \
|
|
--cache-from "${E2E_TESTER_CACHE_REF}" \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
--build-arg PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION}" \
|
|
--build-arg PLAYWRIGHT_DISTRO="${PLAYWRIGHT_DISTRO}" \
|
|
--build-arg PLAYWRIGHT_BASE_IMAGE="${PLAYWRIGHT_BASE_IMAGE}" \
|
|
-t plex-playlist-e2e:"${HEAD_SHA}" .
|
|
docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_TAG_REF}"
|
|
docker tag plex-playlist-e2e:"${HEAD_SHA}" "${E2E_TESTER_CACHE_REF}"
|
|
|
|
retry_registry_op push "${E2E_TESTER_TAG_REF}" 5 4
|
|
retry_registry_op push "${E2E_TESTER_CACHE_REF}" 5 4
|
|
retry_registry_op pull "${E2E_TESTER_TAG_REF}" 5 3
|
|
|
|
E2E_TESTER_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${E2E_TESTER_TAG_REF}" | grep "^${E2E_TESTER_REPO}@sha256:" | head -n 1; } || true)"
|
|
if [ -z "${E2E_TESTER_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve E2E tester digest"
|
|
exit 1
|
|
fi
|
|
|
|
echo "e2e_tester_tag_ref=${E2E_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "e2e_tester_digest_ref=${E2E_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
production-images-complete:
|
|
name: Production Images Complete
|
|
runs-on: ubuntu-act
|
|
needs: [build-release-images, build-tester-images]
|
|
if: always()
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Confirm required production/tester images are complete
|
|
run: |
|
|
set -euo pipefail
|
|
release_images_status="${{ needs['build-release-images'].result }}"
|
|
tester_images_status="${{ needs['build-tester-images'].result }}"
|
|
|
|
echo "build-release-images=${release_images_status}"
|
|
echo "build-tester-images=${tester_images_status}"
|
|
|
|
if [ "${release_images_status}" != "success" ] || [ "${tester_images_status}" != "success" ]; then
|
|
echo "❌ Production Images Complete gate failed"
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Production, integration tester, and E2E tester images are complete"
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
runtime-images-postmortem:
|
|
name: Production Image Failures Postmortem
|
|
# Run on the broader runner pool so we can still capture diagnostics when
|
|
# one of the active production image jobs fails during platform setup.
|
|
runs-on: ubuntu-act
|
|
needs: [build-release-images, build-tester-images]
|
|
if: always() && (needs['build-release-images'].result == 'failure' || needs['build-tester-images'].result == 'failure')
|
|
timeout-minutes: 10
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- name: Collect production image postmortem context
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
set +e
|
|
|
|
echo "=== Postmortem Context ==="
|
|
echo "workflow=${GITHUB_WORKFLOW:-unknown}"
|
|
echo "run_id=${GITHUB_RUN_ID:-unknown}"
|
|
echo "run_number=${GITHUB_RUN_NUMBER:-unknown}"
|
|
echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
|
|
echo "repository=${GITHUB_REPOSITORY:-unknown}"
|
|
echo "server_url=${GITHUB_SERVER_URL:-unknown}"
|
|
echo "build_release_images_result=${{ needs['build-release-images'].result }}"
|
|
echo "build_tester_images_result=${{ needs['build-tester-images'].result }}"
|
|
|
|
echo "=== Local Runner Telemetry (postmortem job host) ==="
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
|
|
echo "=== Registry Reachability Probe ==="
|
|
timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
if [ -z "${GITHUB_TOKEN:-}" ]; then
|
|
echo "No GITHUB_TOKEN available for API diagnostics"
|
|
exit 0
|
|
fi
|
|
|
|
API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
|
|
JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200"
|
|
|
|
echo "=== Actions API Job Summary ==="
|
|
echo "jobs_url=${JOBS_URL}"
|
|
if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then
|
|
echo "Could not fetch job metadata from Actions API"
|
|
exit 0
|
|
fi
|
|
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 -c "$(printf '%s\n' \
|
|
'import json' \
|
|
'from pathlib import Path' \
|
|
'' \
|
|
'path = Path("/tmp/actions-jobs.json")' \
|
|
'payload = json.loads(path.read_text())' \
|
|
'jobs = payload.get("jobs") or payload.get("data") or []' \
|
|
'' \
|
|
'print("job_count=", len(jobs))' \
|
|
'' \
|
|
'def _g(job, *keys):' \
|
|
' for key in keys:' \
|
|
' if key in job and job[key] not in (None, ""):' \
|
|
' return job[key]' \
|
|
' return "unknown"' \
|
|
'' \
|
|
'for job in jobs:' \
|
|
' name = _g(job, "name", "job_name")' \
|
|
' status = _g(job, "status")' \
|
|
' conclusion = _g(job, "conclusion", "result")' \
|
|
' runner = _g(job, "runner_name", "runner")' \
|
|
' started = _g(job, "started_at", "start_time")' \
|
|
' completed = _g(job, "completed_at", "end_time")' \
|
|
' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \
|
|
'' \
|
|
'targets = {' \
|
|
' "Build Release Images",' \
|
|
' "Build Tester Images",' \
|
|
'}' \
|
|
'' \
|
|
'print("=== targeted production image jobs ===")' \
|
|
'for job in jobs:' \
|
|
' name = str(_g(job, "name", "job_name"))' \
|
|
' if name in targets:' \
|
|
' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \
|
|
' if key in job:' \
|
|
' print(f"{key}={job[key]}")' \
|
|
' print("---")')"
|
|
else
|
|
echo "python3 unavailable; emitting raw jobs payload head"
|
|
sed -n '1,120p' /tmp/actions-jobs.json || true
|
|
fi
|
|
|
|
exit 0
|
|
|
|
source-lanes-postmortem:
|
|
name: Source Lanes Failure Postmortem
|
|
# Capture follow-up diagnostics when any source lane dies during platform
|
|
# setup before step logging is available.
|
|
runs-on: ubuntu-act
|
|
needs: [source-checks]
|
|
if: always() && needs['source-checks'].result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- name: Collect source lane postmortem context
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
set +e
|
|
|
|
echo "=== Postmortem Context ==="
|
|
echo "workflow=${GITHUB_WORKFLOW:-unknown}"
|
|
echo "run_id=${GITHUB_RUN_ID:-unknown}"
|
|
echo "run_number=${GITHUB_RUN_NUMBER:-unknown}"
|
|
echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
|
|
echo "repository=${GITHUB_REPOSITORY:-unknown}"
|
|
echo "server_url=${GITHUB_SERVER_URL:-unknown}"
|
|
echo "source_checks_result=${{ needs['source-checks'].result }}"
|
|
|
|
echo "=== Local Runner Telemetry (postmortem job host) ==="
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
|
|
echo "=== Registry Reachability Probe ==="
|
|
timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
if [ -z "${GITHUB_TOKEN:-}" ]; then
|
|
echo "No GITHUB_TOKEN available for API diagnostics"
|
|
exit 0
|
|
fi
|
|
|
|
API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}"
|
|
JOBS_URL="${API_BASE}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?limit=200"
|
|
|
|
echo "=== Actions API Job Summary ==="
|
|
echo "jobs_url=${JOBS_URL}"
|
|
if ! curl -fsSL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${JOBS_URL}" -o /tmp/actions-jobs.json; then
|
|
echo "Could not fetch job metadata from Actions API"
|
|
exit 0
|
|
fi
|
|
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 -c "$(printf '%s\n' \
|
|
'import json' \
|
|
'from pathlib import Path' \
|
|
'' \
|
|
'path = Path("/tmp/actions-jobs.json")' \
|
|
'payload = json.loads(path.read_text())' \
|
|
'jobs = payload.get("jobs") or payload.get("data") or []' \
|
|
'' \
|
|
'print("job_count=", len(jobs))' \
|
|
'' \
|
|
'def _g(job, *keys):' \
|
|
' for key in keys:' \
|
|
' if key in job and job[key] not in (None, ""):' \
|
|
' return job[key]' \
|
|
' return "unknown"' \
|
|
'' \
|
|
'targets = {' \
|
|
' "Source Checks",' \
|
|
'}' \
|
|
'' \
|
|
'for job in jobs:' \
|
|
' name = str(_g(job, "name", "job_name"))' \
|
|
' if name in targets:' \
|
|
' status = _g(job, "status")' \
|
|
' conclusion = _g(job, "conclusion", "result")' \
|
|
' runner = _g(job, "runner_name", "runner")' \
|
|
' started = _g(job, "started_at", "start_time")' \
|
|
' completed = _g(job, "completed_at", "end_time")' \
|
|
' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")')"
|
|
else
|
|
echo "python3 unavailable; emitting raw jobs payload head"
|
|
sed -n '1,120p' /tmp/actions-jobs.json || true
|
|
fi
|
|
|
|
exit 0
|
|
|
|
integration-tests:
|
|
name: Runtime Black-Box Integration Tests
|
|
# Use the broader runner pool; only docker build steps are 8gb-exclusive.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 20
|
|
needs: [build_cicd, production-images-complete, build-release-images, build-tester-images]
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- name: Run runtime black-box integration checks via compose
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }}
|
|
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
|
|
INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.integration_tester_digest_ref }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
RUN_ID="${GITHUB_RUN_ID:-local}"
|
|
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
|
|
LOG_FILE="$(mktemp)"
|
|
COMPOSE_FILE="/tmp/compose.ci.integration.yaml"
|
|
COMPOSE_PROJECT_NAME="plex-int-${RUN_ID}-${RUN_ATTEMPT}"
|
|
DB_PASSWORD="plex_password"
|
|
DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist"
|
|
VERSION_ENV_PATH="/tmp/ci.env"
|
|
RESOLVER_PATH="/tmp/resolve-mirrored-image.sh"
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "❌ Docker is required"
|
|
exit 1
|
|
fi
|
|
if ! docker compose version >/dev/null 2>&1; then
|
|
echo "❌ docker compose plugin is required"
|
|
exit 1
|
|
fi
|
|
|
|
login_ok=false
|
|
for i in 1 2 3 4 5; do
|
|
if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
|
|
login_ok=true
|
|
break
|
|
fi
|
|
if [ "${i}" -lt 5 ]; then
|
|
sleep_seconds=$((3 * i))
|
|
echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
done
|
|
if [ "${login_ok}" != "true" ]; then
|
|
echo "❌ Failed docker login after retries"
|
|
exit 1
|
|
fi
|
|
|
|
RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
|
RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"
|
|
docker cp "${RESOLVER_CONTAINER}:/workspace/scripts/resolve-mirrored-image.sh" "${RESOLVER_PATH}"
|
|
docker cp "${RESOLVER_CONTAINER}:/workspace/versions/ci.env" "${VERSION_ENV_PATH}"
|
|
docker rm -f "${RESOLVER_CONTAINER}" >/dev/null 2>&1 || true
|
|
chmod +x "${RESOLVER_PATH}"
|
|
|
|
set -a
|
|
# shellcheck source=/tmp/ci.env
|
|
source "${VERSION_ENV_PATH}"
|
|
set +a
|
|
|
|
POSTGRES_IMAGE_MIRROR="${GITEA_REGISTRY}/${POSTGRES_MIRROR_IMAGE}"
|
|
POSTGRES_IMAGE_UPSTREAM="${POSTGRES_UPSTREAM_IMAGE}"
|
|
|
|
POSTGRES_IMAGE="$(${RESOLVER_PATH} \
|
|
--image "${POSTGRES_IMAGE_MIRROR}" \
|
|
--mirror-image "${POSTGRES_IMAGE_MIRROR}" \
|
|
--upstream-image "${POSTGRES_IMAGE_UPSTREAM}" \
|
|
--registry-user "${{ github.actor }}" \
|
|
--registry-password-env PACKAGE_ACCESS_TOKEN)"
|
|
|
|
cat >"${COMPOSE_FILE}" <<'EOF'
|
|
services:
|
|
database:
|
|
image: ${POSTGRES_IMAGE}
|
|
environment:
|
|
POSTGRES_DB: plex_playlist
|
|
POSTGRES_USER: plex_user
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 12
|
|
|
|
backend:
|
|
image: ${DEPLOYABLE_BACKEND_IMAGE}
|
|
environment:
|
|
DATABASE_URL: ${DB_URL}
|
|
ENVIRONMENT: production
|
|
depends_on:
|
|
database:
|
|
condition: service_healthy
|
|
EOF
|
|
|
|
fetch_backend_endpoint() {
|
|
endpoint_path="$1"
|
|
output_file="$2"
|
|
probe_tmp="$(mktemp)"
|
|
|
|
if ! docker run --rm \
|
|
--network "${COMPOSE_PROJECT_NAME}_default" \
|
|
"${INTEGRATION_TESTER_DIGEST_REF}" \
|
|
python -c "$(printf '%s\n' \
|
|
'import sys' \
|
|
'import urllib.error' \
|
|
'import urllib.request' \
|
|
'endpoint_path = sys.argv[1]' \
|
|
'url = f"http://backend:8000{endpoint_path}"' \
|
|
'try:' \
|
|
' with urllib.request.urlopen(url, timeout=2) as response:' \
|
|
' body = response.read().decode("utf-8", errors="replace")' \
|
|
' print(response.status)' \
|
|
' print(body)' \
|
|
'except urllib.error.HTTPError as err:' \
|
|
' body = err.read().decode("utf-8", errors="replace")' \
|
|
' print(err.code)' \
|
|
' print(body)' \
|
|
'except Exception:' \
|
|
' print("000")' \
|
|
' print("")')" \
|
|
"${endpoint_path}" >"${probe_tmp}" 2>/dev/null; then
|
|
: >"${output_file}"
|
|
echo "000"
|
|
rm -f "${probe_tmp}"
|
|
return 0
|
|
fi
|
|
|
|
http_code="$(head -n 1 "${probe_tmp}")"
|
|
tail -n +2 "${probe_tmp}" >"${output_file}"
|
|
rm -f "${probe_tmp}"
|
|
echo "${http_code}"
|
|
}
|
|
|
|
dump_failure_context() {
|
|
echo "=== Runtime Integration Failure Context ==="
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true
|
|
echo "--- Compose logs (tail 200) ---"
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true
|
|
}
|
|
|
|
cleanup() {
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "pull" ]; then
|
|
if docker pull "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
return 1
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
{
|
|
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Missing deployable backend image references from dispatch inputs"
|
|
exit 1
|
|
fi
|
|
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
|
|
EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
|
ACTUAL_TAG_DIGEST_REF="$({
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
|
|
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
|
|
| head -n 1
|
|
} || true)"
|
|
|
|
if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then
|
|
echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then
|
|
echo "❌ Tag and digest mismatch"
|
|
echo "expected=${EXPECTED_DIGEST_REF}"
|
|
echo "actual=${ACTUAL_TAG_DIGEST_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}"
|
|
|
|
export DB_PASSWORD DB_URL POSTGRES_IMAGE DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_DIGEST_REF}"
|
|
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend
|
|
|
|
db_ready=false
|
|
for i in $(seq 1 30); do
|
|
if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
|
|
db_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${db_ready}" != "true" ]; then
|
|
echo "❌ Database did not become ready"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
backend_ready=false
|
|
for i in $(seq 1 40); do
|
|
backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
|
|
if [ "${backend_running_count}" -eq 0 ]; then
|
|
echo "❌ Backend container exited before becoming healthy"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
|
if [ "${health_code}" = "200" ]; then
|
|
backend_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${backend_ready}" != "true" ]; then
|
|
echo "❌ Backend did not become healthy"
|
|
cat /tmp/blackbox-health.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)"
|
|
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
|
|
echo "❌ Root endpoint validation failed"
|
|
cat /tmp/blackbox-root.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)"
|
|
if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
|
|
echo "❌ Compatibility endpoint validation failed"
|
|
cat /tmp/blackbox-compatibility.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
|
if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then
|
|
echo "❌ Health endpoint validation failed"
|
|
cat /tmp/blackbox-health.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Runtime integration checks passed"
|
|
} 2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ Runtime integration checks failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of runtime integration output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
integration-tests-postmortem:
|
|
name: Integration Tests Failure Postmortem
|
|
runs-on: ubuntu-act
|
|
needs: integration-tests
|
|
if: always() && needs['integration-tests'].result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Integration postmortem diagnostics
|
|
run: |
|
|
echo "=== Integration Tests Postmortem ==="
|
|
echo "integration_tests_result=${{ needs['integration-tests'].result }}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
e2e-tests:
|
|
name: End-to-End Tests
|
|
# Use ubuntu-act runner pool for consistent availability.
|
|
runs-on: ubuntu-act
|
|
timeout-minutes: 45
|
|
needs: [build_cicd, production-images-complete, build-release-images, build-tester-images]
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- name: Run E2E tests against runtime services via compose
|
|
env:
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }}
|
|
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
|
|
DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_tag_ref }}
|
|
DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }}
|
|
E2E_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.e2e_tester_digest_ref }}
|
|
run: |
|
|
set -euo pipefail
|
|
LOG_FILE="$(mktemp)"
|
|
ARTIFACT_DIR="$(mktemp -d)"
|
|
RUN_ID="${GITHUB_RUN_ID:-local}"
|
|
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
|
|
COMPOSE_FILE="/tmp/compose.ci.e2e.yaml"
|
|
COMPOSE_PROJECT_NAME="plex-e2e-${RUN_ID}-${RUN_ATTEMPT}"
|
|
DB_PASSWORD="plex_password"
|
|
DB_URL="postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist"
|
|
E2E_REPO_DIR="$(mktemp -d)"
|
|
VERSION_ENV_PATH="/tmp/ci.env"
|
|
RESOLVER_PATH="/tmp/resolve-mirrored-image.sh"
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "❌ Docker is required"
|
|
exit 1
|
|
fi
|
|
if ! docker compose version >/dev/null 2>&1; then
|
|
echo "❌ docker compose plugin is required"
|
|
exit 1
|
|
fi
|
|
|
|
login_ok=false
|
|
for i in 1 2 3 4 5; do
|
|
if echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin; then
|
|
login_ok=true
|
|
break
|
|
fi
|
|
if [ "${i}" -lt 5 ]; then
|
|
sleep_seconds=$((3 * i))
|
|
echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
done
|
|
if [ "${login_ok}" != "true" ]; then
|
|
echo "❌ Failed docker login after retries"
|
|
exit 1
|
|
fi
|
|
|
|
RESOLVER_SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
|
RESOLVER_CONTAINER="$(docker create "${RESOLVER_SOURCE_IMAGE}")"
|
|
docker cp "${RESOLVER_CONTAINER}:/workspace/scripts/resolve-mirrored-image.sh" "${RESOLVER_PATH}"
|
|
docker cp "${RESOLVER_CONTAINER}:/workspace/versions/ci.env" "${VERSION_ENV_PATH}"
|
|
docker rm -f "${RESOLVER_CONTAINER}" >/dev/null 2>&1 || true
|
|
chmod +x "${RESOLVER_PATH}"
|
|
|
|
set -a
|
|
# shellcheck source=/tmp/ci.env
|
|
source "${VERSION_ENV_PATH}"
|
|
set +a
|
|
|
|
POSTGRES_IMAGE_MIRROR="${GITEA_REGISTRY}/${POSTGRES_MIRROR_IMAGE}"
|
|
POSTGRES_IMAGE_UPSTREAM="${POSTGRES_UPSTREAM_IMAGE}"
|
|
|
|
POSTGRES_IMAGE="$(${RESOLVER_PATH} \
|
|
--image "${POSTGRES_IMAGE_MIRROR}" \
|
|
--mirror-image "${POSTGRES_IMAGE_MIRROR}" \
|
|
--upstream-image "${POSTGRES_IMAGE_UPSTREAM}" \
|
|
--registry-user "${{ github.actor }}" \
|
|
--registry-password-env PACKAGE_ACCESS_TOKEN)"
|
|
|
|
cat >"${COMPOSE_FILE}" <<'EOF'
|
|
services:
|
|
database:
|
|
image: ${POSTGRES_IMAGE}
|
|
environment:
|
|
POSTGRES_DB: plex_playlist
|
|
POSTGRES_USER: plex_user
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 12
|
|
|
|
backend:
|
|
image: ${DEPLOYABLE_BACKEND_IMAGE}
|
|
environment:
|
|
DATABASE_URL: ${DB_URL}
|
|
ENVIRONMENT: production
|
|
depends_on:
|
|
database:
|
|
condition: service_healthy
|
|
|
|
frontend:
|
|
image: ${DEPLOYABLE_FRONTEND_IMAGE}
|
|
depends_on:
|
|
backend:
|
|
condition: service_started
|
|
EOF
|
|
|
|
fetch_http() {
|
|
container_name="$1"
|
|
endpoint_path="$2"
|
|
output_file="$3"
|
|
probe_tmp="$(mktemp)"
|
|
|
|
run_probe_with_python() {
|
|
python_bin="$1"
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T "${container_name}" "${python_bin}" -c "$(printf '%s\n' \
|
|
'import sys' \
|
|
'import urllib.error' \
|
|
'import urllib.request' \
|
|
'endpoint_path = sys.argv[1]' \
|
|
'url = f"http://127.0.0.1:8000{endpoint_path}"' \
|
|
'try:' \
|
|
' with urllib.request.urlopen(url, timeout=2) as response:' \
|
|
' body = response.read().decode("utf-8", errors="replace")' \
|
|
' print(response.status)' \
|
|
' print(body)' \
|
|
'except urllib.error.HTTPError as err:' \
|
|
' body = err.read().decode("utf-8", errors="replace")' \
|
|
' print(err.code)' \
|
|
' print(body)' \
|
|
'except Exception:' \
|
|
' print("000")' \
|
|
' print("")')" "${endpoint_path}"
|
|
}
|
|
|
|
probe_err="$(mktemp)"
|
|
|
|
if run_probe_with_python python >"${probe_tmp}" 2>"${probe_err}" || run_probe_with_python python3 >"${probe_tmp}" 2>>"${probe_err}"; then
|
|
:
|
|
else
|
|
: >"${output_file}"
|
|
{
|
|
echo '{"status":"probe_exec_failed"}'
|
|
sed -n '1,5p' "${probe_err}" || true
|
|
} >"${output_file}"
|
|
rm -f "${probe_tmp}"
|
|
rm -f "${probe_err}"
|
|
echo "000"
|
|
return 0
|
|
fi
|
|
|
|
http_code="$(head -n 1 "${probe_tmp}")"
|
|
tail -n +2 "${probe_tmp}" >"${output_file}"
|
|
rm -f "${probe_tmp}"
|
|
rm -f "${probe_err}"
|
|
echo "${http_code}"
|
|
}
|
|
|
|
dump_failure_context() {
|
|
echo "=== Runtime E2E Failure Context ==="
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps || true
|
|
echo "--- Compose logs (tail 200) ---"
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --no-color --tail=200 || true
|
|
echo "--- Artifact directory ---"
|
|
find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
|
|
}
|
|
|
|
cleanup() {
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
|
|
rm -rf "${E2E_REPO_DIR}" >/dev/null 2>&1 || true
|
|
rm -f ~/.ssh/id_rsa >/dev/null 2>&1 || true
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "pull" ]; then
|
|
if docker pull "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
return 1
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Missing deployable runtime image references from dispatch inputs"
|
|
exit 1
|
|
fi
|
|
|
|
SOURCE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
|
SOURCE_CONTAINER="$(docker create "${SOURCE_IMAGE}")"
|
|
docker cp "${SOURCE_CONTAINER}:/workspace/frontend" "${E2E_REPO_DIR}/frontend"
|
|
docker rm -f "${SOURCE_CONTAINER}" >/dev/null 2>&1 || true
|
|
|
|
if [ ! -f "${E2E_REPO_DIR}/frontend/package.json" ]; then
|
|
echo "❌ Missing frontend/package.json in cloned E2E workspace"
|
|
find "${E2E_REPO_DIR}" -maxdepth 3 -type f | sort || true
|
|
exit 1
|
|
fi
|
|
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_BACKEND_DIGEST_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
|
|
retry_registry_op pull "${DEPLOYABLE_FRONTEND_DIGEST_REF}" 5 3
|
|
|
|
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
|
|
EXPECTED_BACKEND_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
|
ACTUAL_BACKEND_TAG_DIGEST_REF="$({
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
|
|
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
|
|
| head -n 1
|
|
} || true)"
|
|
|
|
if [ -z "${ACTUAL_BACKEND_TAG_DIGEST_REF}" ]; then
|
|
echo "❌ Could not resolve backend digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${ACTUAL_BACKEND_TAG_DIGEST_REF}" != "${EXPECTED_BACKEND_DIGEST_REF}" ]; then
|
|
echo "❌ Backend tag and digest mismatch"
|
|
echo "expected=${EXPECTED_BACKEND_DIGEST_REF}"
|
|
echo "actual=${ACTUAL_BACKEND_TAG_DIGEST_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
DEPLOYABLE_FRONTEND_REPO="${DEPLOYABLE_FRONTEND_DIGEST_REF%%@*}"
|
|
EXPECTED_FRONTEND_DIGEST_REF="${DEPLOYABLE_FRONTEND_DIGEST_REF}"
|
|
ACTUAL_FRONTEND_TAG_DIGEST_REF="$({
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" \
|
|
| grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" \
|
|
| head -n 1
|
|
} || true)"
|
|
|
|
if [ -z "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" ]; then
|
|
echo "❌ Could not resolve frontend digest from tag reference: ${DEPLOYABLE_FRONTEND_TAG_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${ACTUAL_FRONTEND_TAG_DIGEST_REF}" != "${EXPECTED_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Frontend tag and digest mismatch"
|
|
echo "expected=${EXPECTED_FRONTEND_DIGEST_REF}"
|
|
echo "actual=${ACTUAL_FRONTEND_TAG_DIGEST_REF}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Resolved deployable backend digest: ${EXPECTED_BACKEND_DIGEST_REF}"
|
|
echo "Resolved deployable frontend digest: ${EXPECTED_FRONTEND_DIGEST_REF}"
|
|
|
|
export DB_PASSWORD DB_URL POSTGRES_IMAGE E2E_ARTIFACT_DIR="${ARTIFACT_DIR}"
|
|
export DEPLOYABLE_BACKEND_IMAGE="${EXPECTED_BACKEND_DIGEST_REF}"
|
|
export DEPLOYABLE_FRONTEND_IMAGE="${EXPECTED_FRONTEND_DIGEST_REF}"
|
|
|
|
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend
|
|
|
|
db_ready=false
|
|
for i in $(seq 1 30); do
|
|
if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T database pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
|
|
db_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${db_ready}" != "true" ]; then
|
|
echo "❌ Database did not become ready"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
backend_ready=false
|
|
for i in $(seq 1 40); do
|
|
backend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
|
|
if [ "${backend_running_count}" -eq 0 ]; then
|
|
echo "❌ Backend container exited before becoming healthy"
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
health_code="$(fetch_http "backend" "/health" /tmp/e2e-backend-health.json)"
|
|
if [ "${health_code}" = "200" ]; then
|
|
backend_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${backend_ready}" != "true" ]; then
|
|
echo "❌ Backend did not become healthy"
|
|
cat /tmp/e2e-backend-health.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
root_code="$(fetch_http "backend" "/" /tmp/e2e-backend-root.json)"
|
|
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/e2e-backend-root.json; then
|
|
echo "❌ Backend root endpoint validation failed"
|
|
cat /tmp/e2e-backend-root.json 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
frontend_ready=false
|
|
for i in $(seq 1 30); do
|
|
frontend_running_count="$(docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services frontend | wc -l)"
|
|
if [ "${frontend_running_count}" -gt 0 ]; then
|
|
if docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T frontend sh -lc 'wget -q -T 2 -O /dev/null http://127.0.0.1/'; then
|
|
frontend_ready=true
|
|
break
|
|
fi
|
|
fi
|
|
sleep 2
|
|
done
|
|
if [ "${frontend_ready}" != "true" ]; then
|
|
echo "❌ Frontend did not become reachable"
|
|
cat /tmp/e2e-frontend-root.html 2>/dev/null || true
|
|
dump_failure_context
|
|
exit 1
|
|
fi
|
|
|
|
tar -C "${E2E_REPO_DIR}" -cf - . \
|
|
| docker run --rm -i \
|
|
--network "${COMPOSE_PROJECT_NAME}_default" \
|
|
-e CI=true \
|
|
-e PLAYWRIGHT_BASE_URL="http://frontend:80" \
|
|
-e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \
|
|
-e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \
|
|
-v "${ARTIFACT_DIR}:/tmp/playwright-artifacts" \
|
|
"${E2E_TESTER_DIGEST_REF}" \
|
|
bash -lc 'set -euo pipefail; retry_cmd(){ attempts="${1:-5}"; backoff="${2:-3}"; shift 2; attempt=1; while [ "${attempt}" -le "${attempts}" ]; do if "$@"; then return 0; fi; if [ "${attempt}" -lt "${attempts}" ]; then sleep_seconds=$((backoff * attempt)); echo "Retrying in ${sleep_seconds}s"; sleep "${sleep_seconds}"; fi; attempt=$((attempt + 1)); done; return 1; }; mkdir -p /workspace; tar -xf - -C /workspace; cd /workspace/frontend; if [ ! -f package.json ]; then echo "❌ Missing package.json in /workspace/frontend"; ls -la /workspace/frontend; find /workspace -maxdepth 3 -type f | sort | head -n 100; exit 1; fi; retry_cmd 5 4 corepack yarn install --immutable; corepack yarn test:e2e --reporter=list --timeout=90000' \
|
|
2>&1 | tee "${LOG_FILE}"
|
|
|
|
TEST_STATUS=${PIPESTATUS[0]}
|
|
if [ "${TEST_STATUS}" -ne 0 ]; then
|
|
echo "❌ E2E tests failed (exit=${TEST_STATUS})"
|
|
echo "--- Last 200 lines of E2E output ---"
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
echo "--- E2E artifact files ---"
|
|
find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
|
|
dump_failure_context
|
|
exit "${TEST_STATUS}"
|
|
fi
|
|
|
|
echo "=== E2E artifact files ==="
|
|
find "${ARTIFACT_DIR}" -maxdepth 2 -type f | sort || true
|
|
|
|
- *failure_diagnostics_step
|
|
|
|
e2e-tests-postmortem:
|
|
name: E2E Tests Failure Postmortem
|
|
runs-on: ubuntu-act
|
|
needs: e2e-tests
|
|
if: always() && needs['e2e-tests'].result == 'failure'
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: E2E postmortem diagnostics
|
|
run: |
|
|
echo "=== E2E Tests Postmortem ==="
|
|
echo "e2e_tests_result=${{ needs['e2e-tests'].result }}"
|
|
uname -a || true
|
|
cat /etc/os-release 2>/dev/null || true
|
|
df -h || true
|
|
free -h || true
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker version || true
|
|
docker info || true
|
|
fi
|
|
timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true
|
|
|
|
promote-release-images:
|
|
name: Promote Staging Images To Release
|
|
runs-on: ubuntu-act
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
timeout-minutes: 15
|
|
needs: [build_cicd, build-release-images, integration-tests, e2e-tests]
|
|
outputs:
|
|
release_version: ${{ steps.promote.outputs.release_version }}
|
|
release_short_sha: ${{ steps.promote.outputs.release_short_sha }}
|
|
release_tagged_version: ${{ steps.promote.outputs.release_tagged_version }}
|
|
previous_release_version: ${{ steps.promote.outputs.previous_release_version }}
|
|
backend_release_version_ref: ${{ steps.promote.outputs.backend_release_version_ref }}
|
|
backend_release_tagged_ref: ${{ steps.promote.outputs.backend_release_tagged_ref }}
|
|
backend_release_build_ref: ${{ steps.promote.outputs.backend_release_build_ref }}
|
|
frontend_release_version_ref: ${{ steps.promote.outputs.frontend_release_version_ref }}
|
|
frontend_release_tagged_ref: ${{ steps.promote.outputs.frontend_release_tagged_ref }}
|
|
frontend_release_build_ref: ${{ steps.promote.outputs.frontend_release_build_ref }}
|
|
steps:
|
|
- *identify_runner_step
|
|
|
|
- *configure_registry_host_step
|
|
- name: Promote staging images to release tags
|
|
id: promote
|
|
env:
|
|
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
|
REGISTRY_USER: ${{ github.actor }}
|
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
|
|
STAGING_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
|
|
STAGING_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }}
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
TMP_REPO_DIR="$(mktemp -d)"
|
|
trap 'rm -f /tmp/release_ssh_key; rm -rf "${TMP_REPO_DIR}"' EXIT
|
|
|
|
HEAD_SHA="${HEAD_SHA:-${GITHUB_SHA:-}}"
|
|
if [ -z "${HEAD_SHA}" ]; then
|
|
echo "❌ Unable to resolve HEAD_SHA"
|
|
exit 1
|
|
fi
|
|
|
|
retry_cmd() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-2}"
|
|
shift 2
|
|
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Command failed (attempt ${attempt}/${attempts}): $*"
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
echo "Command failed after ${attempts} attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
docker_login_with_retry() {
|
|
attempts="${1:-5}"
|
|
backoff="${2:-3}"
|
|
attempt=1
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
|
return 0
|
|
fi
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "docker login failed; retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
retry_registry_op() {
|
|
op_name="$1"
|
|
image_ref="$2"
|
|
attempts="${3:-5}"
|
|
backoff="${4:-3}"
|
|
attempt=1
|
|
|
|
while [ "${attempt}" -le "${attempts}" ]; do
|
|
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
|
|
if [ "${op_name}" = "push" ]; then
|
|
docker_login_with_retry 3 2
|
|
if docker push "${image_ref}"; then
|
|
return 0
|
|
fi
|
|
else
|
|
if docker pull "${image_ref}" >/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if [ "${attempt}" -lt "${attempts}" ]; then
|
|
sleep_seconds=$((backoff * attempt))
|
|
echo "Retrying in ${sleep_seconds}s"
|
|
sleep "${sleep_seconds}"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
resolve_staging_digest_from_tag() {
|
|
image_repo="$1"
|
|
image_tag="$2"
|
|
image_tag_ref="${image_repo}:${image_tag}"
|
|
|
|
retry_registry_op pull "${image_tag_ref}" 5 3
|
|
digest_ref="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${image_tag_ref}" | grep "^${image_repo}@sha256:" | head -n 1; } || true)"
|
|
printf '%s' "${digest_ref}"
|
|
}
|
|
|
|
docker_login_with_retry 5 3
|
|
|
|
if [ -z "${STAGING_BACKEND_DIGEST_REF}" ]; then
|
|
echo "Staging backend digest output missing; deriving from staging tag ${HEAD_SHA}"
|
|
STAGING_BACKEND_DIGEST_REF="$(resolve_staging_digest_from_tag "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend-staging" "${HEAD_SHA}")"
|
|
fi
|
|
|
|
if [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "Staging frontend digest output missing; deriving from staging tag ${HEAD_SHA}"
|
|
STAGING_FRONTEND_DIGEST_REF="$(resolve_staging_digest_from_tag "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend-staging" "${HEAD_SHA}")"
|
|
fi
|
|
|
|
if [ -z "${STAGING_BACKEND_DIGEST_REF}" ] || [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then
|
|
echo "❌ Unable to resolve staging image digests for HEAD_SHA=${HEAD_SHA}"
|
|
echo "backend_staging_digest_ref=${STAGING_BACKEND_DIGEST_REF:-}"
|
|
echo "frontend_staging_digest_ref=${STAGING_FRONTEND_DIGEST_REF:-}"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
echo "${SSH_PRIVATE_KEY}" > /tmp/release_ssh_key
|
|
chmod 600 /tmp/release_ssh_key
|
|
ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
GIT_SSH_COMMAND="ssh -i /tmp/release_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
|
|
|
|
SHORT_SHA="$(printf '%s' "${HEAD_SHA}" | cut -c1-7)"
|
|
|
|
HEAD_SEMVER_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
|
|
| awk -v sha="${HEAD_SHA}" '$1==sha {tag=$2; sub("refs/tags/", "", tag); if (tag ~ /^v[0-9]+\.[0-9]+\.[0-9]+$/) print tag}' \
|
|
| sort -V \
|
|
| tail -n 1)"
|
|
|
|
if [ -z "${HEAD_SEMVER_TAG}" ]; then
|
|
ALL_SEMVER_TAGS="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
|
|
| awk '{tag=$2; sub("refs/tags/", "", tag); if (tag ~ /^v[0-9]+\.[0-9]+\.[0-9]+$/) print tag}' \
|
|
| sort -V)"
|
|
|
|
LAST_SEMVER_TAG="$(printf '%s\n' "${ALL_SEMVER_TAGS}" | tail -n 1)"
|
|
if [ -z "${LAST_SEMVER_TAG}" ]; then
|
|
LAST_SEMVER_TAG="v0.0.0"
|
|
fi
|
|
|
|
TAG_MAJOR="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')"
|
|
TAG_MINOR="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')"
|
|
TAG_PATCH="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\3/')"
|
|
NEXT_PATCH="$((TAG_PATCH + 1))"
|
|
HEAD_SEMVER_TAG="v${TAG_MAJOR}.${TAG_MINOR}.${NEXT_PATCH}"
|
|
|
|
retry_cmd 5 2 git -C "${TMP_REPO_DIR}" init -q
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" remote add origin "${GITEA_REPO_SSH_URL}"
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --no-tags --depth 200 origin "${HEAD_SHA}"
|
|
git -C "${TMP_REPO_DIR}" config user.name "gitea-actions"
|
|
git -C "${TMP_REPO_DIR}" config user.email "gitea-actions@darkhelm.local"
|
|
git -C "${TMP_REPO_DIR}" tag "${HEAD_SEMVER_TAG}" "${HEAD_SHA}"
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" push origin "refs/tags/${HEAD_SEMVER_TAG}"
|
|
echo "Created patch release tag ${HEAD_SEMVER_TAG} for ${HEAD_SHA}"
|
|
fi
|
|
|
|
TAG_MAJOR="$(printf '%s' "${HEAD_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')"
|
|
TAG_MINOR="$(printf '%s' "${HEAD_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')"
|
|
RELEASE_VERSION="v${TAG_MAJOR}.${TAG_MINOR}.0"
|
|
RELEASE_TAGGED_VERSION="${HEAD_SEMVER_TAG}"
|
|
RELEASE_BUILD_TAG="${RELEASE_TAGGED_VERSION}-${SHORT_SHA}"
|
|
|
|
RELEASE_TAG_LIST="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
|
|
| awk '{tag=$2; sub("refs/tags/", "", tag); if (tag ~ /^v[0-9]+\.[0-9]+\.0$/) print tag}' \
|
|
| sort -V)"
|
|
|
|
PREVIOUS_RELEASE_VERSION="$(printf '%s\n' "${RELEASE_TAG_LIST}" | awk -v current="${RELEASE_VERSION}" 'NF {if ($0 == current) {print prev; found=1; exit} prev=$0} END {if (!found) print prev}')"
|
|
|
|
RELEASE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-backend"
|
|
RELEASE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-frontend"
|
|
|
|
BACKEND_RELEASE_VERSION_REF="${RELEASE_BACKEND_REPO}:${RELEASE_VERSION}"
|
|
BACKEND_RELEASE_TAGGED_REF="${RELEASE_BACKEND_REPO}:${RELEASE_TAGGED_VERSION}"
|
|
BACKEND_RELEASE_BUILD_REF="${RELEASE_BACKEND_REPO}:${RELEASE_BUILD_TAG}"
|
|
BACKEND_RELEASE_LATEST_REF="${RELEASE_BACKEND_REPO}:latest"
|
|
FRONTEND_RELEASE_VERSION_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_VERSION}"
|
|
FRONTEND_RELEASE_TAGGED_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_TAGGED_VERSION}"
|
|
FRONTEND_RELEASE_BUILD_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_BUILD_TAG}"
|
|
FRONTEND_RELEASE_LATEST_REF="${RELEASE_FRONTEND_REPO}:latest"
|
|
|
|
if [ ! -d "${TMP_REPO_DIR}/.git" ]; then
|
|
retry_cmd 5 2 git -C "${TMP_REPO_DIR}" init -q
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" remote add origin "${GITEA_REPO_SSH_URL}"
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --no-tags --depth 200 origin "${HEAD_SHA}"
|
|
fi
|
|
retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --tags --force origin
|
|
|
|
RELEASE_NOTES_FILE="$(mktemp)"
|
|
PREVIOUS_RELEASE_GIT_REF="${PREVIOUS_RELEASE_VERSION}"
|
|
if [ -z "${PREVIOUS_RELEASE_GIT_REF}" ]; then
|
|
PREVIOUS_RELEASE_VERSION="v0.0.0"
|
|
fi
|
|
|
|
if [ -n "${PREVIOUS_RELEASE_GIT_REF}" ]; then
|
|
LOG_RANGE="${PREVIOUS_RELEASE_VERSION}..${HEAD_SHA}"
|
|
CHANGES_HEADER="Changes since previous release ${PREVIOUS_RELEASE_VERSION}:"
|
|
else
|
|
LOG_RANGE="${HEAD_SHA}"
|
|
CHANGES_HEADER="Changes since baseline ${PREVIOUS_RELEASE_VERSION}:"
|
|
fi
|
|
|
|
{
|
|
echo "## Release ${RELEASE_VERSION}"
|
|
echo
|
|
echo "- Release series tag: ${RELEASE_VERSION}"
|
|
echo "- Version tag on commit: ${RELEASE_TAGGED_VERSION}"
|
|
echo "- Build tag: ${RELEASE_BUILD_TAG}"
|
|
if [ -n "${PREVIOUS_RELEASE_VERSION}" ]; then
|
|
echo "- Previous release: ${PREVIOUS_RELEASE_VERSION}"
|
|
else
|
|
echo "- Previous release: (none)"
|
|
fi
|
|
echo
|
|
echo "${CHANGES_HEADER}"
|
|
git -C "${TMP_REPO_DIR}" log --no-merges --pretty='- %s (%h)' "${LOG_RANGE}" || true
|
|
} > "${RELEASE_NOTES_FILE}"
|
|
|
|
if ! grep -q '^- ' "${RELEASE_NOTES_FILE}"; then
|
|
{
|
|
echo
|
|
echo "- No commit-summary entries were found in the selected range."
|
|
} >> "${RELEASE_NOTES_FILE}"
|
|
fi
|
|
|
|
{
|
|
echo "### Generated Release Notes"
|
|
cat "${RELEASE_NOTES_FILE}"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
docker_login_with_retry 5 3
|
|
retry_registry_op pull "${STAGING_BACKEND_DIGEST_REF}" 5 3
|
|
retry_registry_op pull "${STAGING_FRONTEND_DIGEST_REF}" 5 3
|
|
|
|
BACKEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_BACKEND_DIGEST_REF}")"
|
|
FRONTEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_FRONTEND_DIGEST_REF}")"
|
|
|
|
docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_VERSION_REF}"
|
|
docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_TAGGED_REF}"
|
|
docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_BUILD_REF}"
|
|
docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_LATEST_REF}"
|
|
docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_VERSION_REF}"
|
|
docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_TAGGED_REF}"
|
|
docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_BUILD_REF}"
|
|
docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_LATEST_REF}"
|
|
|
|
retry_registry_op push "${BACKEND_RELEASE_VERSION_REF}" 5 4
|
|
retry_registry_op push "${BACKEND_RELEASE_TAGGED_REF}" 5 4
|
|
retry_registry_op push "${BACKEND_RELEASE_BUILD_REF}" 5 4
|
|
retry_registry_op push "${BACKEND_RELEASE_LATEST_REF}" 5 4
|
|
retry_registry_op push "${FRONTEND_RELEASE_VERSION_REF}" 5 4
|
|
retry_registry_op push "${FRONTEND_RELEASE_TAGGED_REF}" 5 4
|
|
retry_registry_op push "${FRONTEND_RELEASE_BUILD_REF}" 5 4
|
|
retry_registry_op push "${FRONTEND_RELEASE_LATEST_REF}" 5 4
|
|
|
|
echo "release_version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
|
|
echo "release_short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
|
echo "release_tagged_version=${RELEASE_TAGGED_VERSION}" >> "$GITHUB_OUTPUT"
|
|
echo "previous_release_version=${PREVIOUS_RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
|
|
echo "backend_release_version_ref=${BACKEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "backend_release_tagged_ref=${BACKEND_RELEASE_TAGGED_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "backend_release_build_ref=${BACKEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "frontend_release_version_ref=${FRONTEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "frontend_release_tagged_ref=${FRONTEND_RELEASE_TAGGED_REF}" >> "$GITHUB_OUTPUT"
|
|
echo "frontend_release_build_ref=${FRONTEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT"
|
|
|
|
- *failure_diagnostics_step
|