DarkHelm.org/plex-playlist
Public Access
4
1
Fork 0
Code Issues 55 Pull Requests Actions 9 Packages Projects 1 Releases Wiki Activity

ci: consolidate cicd lanes and align docs
Some checks failed
CICD / Build and Push CICD Images (pull_request) Failing after 12m53s
Details
CICD / Source Checks (pull_request) Has been skipped
Details
CICD / Dependency Audits (Informational) (pull_request) Has been skipped
Details
CICD / Build Release Images (pull_request) Has been skipped
Details
CICD / Build Tester Images (pull_request) Has been skipped
Details
CICD / CICD Tests Complete (pull_request) Failing after 7s
Details
CICD / Build CICD Image Failure Postmortem (pull_request) Successful in 9s
Details
CICD / Production Image Failures Postmortem (pull_request) Has been skipped
Details
CICD / Source Lanes Failure Postmortem (pull_request) Has been skipped
Details
CICD / Production Images Complete (pull_request) Failing after 5s
Details
CICD / Runtime Black-Box Integration Tests (pull_request) Has been skipped
Details
CICD / End-to-End Tests (pull_request) Has been skipped
Details
CICD / Integration Tests Failure Postmortem (pull_request) Has been skipped
Details
CICD / E2E Tests Failure Postmortem (pull_request) Has been skipped
Details

Browse Source
This commit is contained in:
copilotcoder
2026-07-16 13:40:30 -04:00
parent ff51f04c36
commit 3d045f71f6
3 changed files with 274 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

649
.gitea/workflows/cicd.yaml
View File

@@ -34,8 +34,8 @@ concurrency:
cancel-in-progress: true
jobs:
publish-base:
name: Build and Publish CICD Base Image
build_cicd:
name: Build and Push CICD Images
runs-on: ubuntu-act-8gb
timeout-minutes: 35
outputs:
@@ -190,25 +190,6 @@ jobs:
return 1
}
docker_login_with_retry() {
attempts="${1:-5}"
backoff="${2:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
ensure_registry_auth_realm_host
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
return 0
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "docker login failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest"
@@ -223,19 +204,7 @@ jobs:
ensure_registry_auth_realm_host
login_ok=false
for i in 1 2 3 4 5; do
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null; then
login_ok=true
break
fi
if [ "${i}" -lt 5 ]; then
sleep_seconds=$((3 * i))
echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
done
if [ "${login_ok}" != "true" ]; then
if ! docker_login_with_retry 5 3 >/dev/null; then
echo "❌ Failed docker login after retries"
exit 1
fi
@@ -365,43 +334,6 @@ jobs:
retry_registry_op push "${BASE_REF_HASH}" 5 4
retry_registry_op push "${BASE_REF_LATEST}" 5 4
- &failure_diagnostics_step
name: Failure diagnostics
if: failure()
run: |
echo "=== Failure Diagnostics ==="
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
echo "runner_name=${RUNNER_NAME:-unknown}"
echo "runner_hostname=${HOSTNAME:-unknown}"
uname -a || true
cat /etc/os-release 2>/dev/null || true
df -h || true
free -h || true
ps aux --sort=-%mem | head -n 30 || true
if command -v docker >/dev/null 2>&1; then
echo "=== Docker Diagnostics ==="
docker version || true
docker info || true
docker ps -a || true
docker images --digests | head -n 50 || true
fi
dmesg | tail -n 120 || true
build_cicd:
name: Build and Push CICD Image
runs-on: ubuntu-act-8gb
needs: publish-base
timeout-minutes: 60
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
steps:
- name: Resolve head SHA
id: meta
run: |
echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT"
- *pre_job_prune_step
- name: Minimal checkout for CICD build inputs
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
@@ -462,7 +394,7 @@ jobs:
REGISTRY_USER: ${{ github.actor }}
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }}
BASE_HASH_FROM_BUILD: ${{ steps.base-state.outputs.base_hash }}
run: |
set -euo pipefail
umask 077
@@ -620,7 +552,27 @@ jobs:
retry_registry_op push "${CICD_LATEST_REF}" 5 4
retry_registry_op push "${CICD_SHA_REF}" 5 4
- *failure_diagnostics_step
- &failure_diagnostics_step
name: Failure diagnostics
if: failure()
run: |
echo "=== Failure Diagnostics ==="
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
echo "runner_name=${RUNNER_NAME:-unknown}"
echo "runner_hostname=${HOSTNAME:-unknown}"
uname -a || true
cat /etc/os-release 2>/dev/null || true
df -h || true
free -h || true
ps aux --sort=-%mem | head -n 30 || true
if command -v docker >/dev/null 2>&1; then
echo "=== Docker Diagnostics ==="
docker version || true
docker info || true
docker ps -a || true
docker images --digests | head -n 50 || true
fi
dmesg | tail -n 120 || true
build-cicd-postmortem:
name: Build CICD Image Failure Postmortem
@@ -831,8 +783,8 @@ jobs:
- *failure_diagnostics_step
frontend-audit:
name: Frontend Dependency Audit
dependency-audits:
name: Dependency Audits (Informational)
runs-on: ubuntu-act
timeout-minutes: 15
continue-on-error: true
@@ -862,19 +814,6 @@ jobs:
echo "Proceeding: frontend audit is informational-only."
fi
- *failure_diagnostics_step
backend-audit:
name: Backend Dependency Audit
runs-on: ubuntu-act
timeout-minutes: 15
continue-on-error: true
needs: build_cicd
steps:
- *identify_runner_step
- *configure_registry_host_step
- *ensure_cicd_image_step
- name: Run backend dependency audit
env:
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
@@ -900,26 +839,24 @@ jobs:
cicd-tests-complete:
name: CICD Tests Complete
runs-on: ubuntu-act
needs: [source-checks, frontend-audit, backend-audit]
needs: [source-checks, dependency-audits]
if: always()
steps:
- name: Confirm all source lanes passed
run: |
set -euo pipefail
source_checks_status="${{ needs['source-checks'].result }}"
frontend_audit_status="${{ needs['frontend-audit'].result }}"
backend_audit_status="${{ needs['backend-audit'].result }}"
dependency_audits_status="${{ needs['dependency-audits'].result }}"
echo "source-checks=${source_checks_status}"
echo "frontend-audit=${frontend_audit_status}"
echo "backend-audit=${backend_audit_status}"
echo "dependency-audits=${dependency_audits_status}"
if [ "${source_checks_status}" != "success" ]; then
echo "❌ One or more CICD source lanes failed"
exit 1
fi
if [ "${frontend_audit_status}" != "success" ] || [ "${backend_audit_status}" != "success" ]; then
if [ "${dependency_audits_status}" != "success" ]; then
echo "⚠️ Dependency audit lane reported non-success (informational only)"
fi
@@ -927,13 +864,20 @@ jobs:
- *failure_diagnostics_step
build-backend-base-image:
name: Build Backend Base Image
build-release-images:
name: Build Release Images
runs-on: ubuntu-act-8gb
needs: [build_cicd, source-checks]
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }}
backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }}
frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }}
frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }}
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
steps:
- &resolve_head_sha_from_build_step
name: Resolve head SHA
@@ -1042,21 +986,6 @@ jobs:
echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- *failure_diagnostics_step
build-frontend-base-image:
name: Build Frontend Base Image
runs-on: ubuntu-act-8gb
needs: [build_cicd, source-checks]
outputs:
frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }}
frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }}
steps:
- *resolve_head_sha_from_build_step
- *configure_registry_host_step
- *ensure_cicd_image_step
- *hydrate_build_context_from_cicd_step
- name: Build and push frontend base image
id: frontend_base_ref
env:
@@ -1151,15 +1080,212 @@ jobs:
echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- name: Build and push backend main image
id: deployable_backend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
docker_login_with_retry() {
attempts="${1:-5}"
backoff="${2:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
return 0
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "docker login failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
retry_registry_op() {
op_name="$1"
image_ref="$2"
attempts="${3:-5}"
backoff="${4:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
if [ "${op_name}" = "push" ]; then
docker_login_with_retry 3 2
if docker push "${image_ref}"; then
return 0
fi
else
if docker pull "${image_ref}" >/dev/null; then
return 0
fi
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache"
docker_login_with_retry 5 3
retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true
DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
--cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-t deployable-backend:"${HEAD_SHA}" .
bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}"
retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve backend main digest"
exit 1
fi
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- name: Build and push frontend main image
id: deployable_frontend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
ensure_registry_auth_realm_host() {
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
headers_file="$(mktemp)"
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
if [ -n "${realm_url}" ]; then
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
if ! grep -q "${realm_host}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
fi
fi
fi
fi
rm -f "${headers_file}"
}
docker_login_with_retry() {
attempts="${1:-5}"
backoff="${2:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
ensure_registry_auth_realm_host
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
return 0
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "docker login failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
retry_registry_op() {
op_name="$1"
image_ref="$2"
attempts="${3:-5}"
backoff="${4:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
if [ "${op_name}" = "push" ]; then
docker_login_with_retry 3 2
if docker push "${image_ref}"; then
return 0
fi
else
if docker pull "${image_ref}" >/dev/null; then
return 0
fi
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache"
docker_login_with_retry 5 3
retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true
BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
--target production \
--cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-t deployable-frontend:"${HEAD_SHA}" .
bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}"
retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve frontend main digest"
exit 1
fi
echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- *failure_diagnostics_step
build-integration-tester-image:
name: Build Integration Tester Image
build-tester-images:
name: Build Tester Images
runs-on: ubuntu-act-8gb
needs: [build_cicd, source-checks]
outputs:
integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }}
integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }}
e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }}
e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }}
steps:
- *resolve_head_sha_from_build_step
- *configure_registry_host_step
@@ -1272,21 +1398,6 @@ jobs:
echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- *failure_diagnostics_step
build-e2e-tester-image:
name: Build E2E Tester Image
runs-on: ubuntu-act-8gb
needs: [build_cicd, source-checks]
outputs:
e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }}
e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }}
steps:
- *resolve_head_sha_from_build_step
- *configure_registry_host_step
- *ensure_cicd_image_step
- *hydrate_build_context_from_cicd_step
- name: Build and push E2E tester image
id: e2e_tester_ref
env:
@@ -1417,254 +1528,23 @@ jobs:
- *failure_diagnostics_step
build-backend-main-image:
name: Build Backend Main Image
runs-on: ubuntu-act-8gb
needs: [build_cicd, source-checks, build-backend-base-image]
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
steps:
- *resolve_head_sha_from_build_step
- *configure_registry_host_step
- *ensure_cicd_image_step
- *hydrate_build_context_from_cicd_step
- name: Build and push backend main image
id: deployable_backend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
docker_login_with_retry() {
attempts="${1:-5}"
backoff="${2:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
return 0
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "docker login failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
retry_registry_op() {
op_name="$1"
image_ref="$2"
attempts="${3:-5}"
backoff="${4:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
if [ "${op_name}" = "push" ]; then
docker_login_with_retry 3 2
if docker push "${image_ref}"; then
return 0
fi
else
if docker pull "${image_ref}" >/dev/null; then
return 0
fi
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache"
docker_login_with_retry 5 3
retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true
DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \
--cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-t deployable-backend:"${HEAD_SHA}" .
bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}"
retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve backend main digest"
exit 1
fi
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- *failure_diagnostics_step
build-frontend-main-image:
name: Build Frontend Main Image
runs-on: ubuntu-act-8gb
needs: [build_cicd, source-checks, build-frontend-base-image]
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
steps:
- *resolve_head_sha_from_build_step
- *configure_registry_host_step
- *ensure_cicd_image_step
- *hydrate_build_context_from_cicd_step
- name: Build and push frontend main image
id: deployable_frontend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
ensure_registry_auth_realm_host() {
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
headers_file="$(mktemp)"
if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then
realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)"
if [ -n "${realm_url}" ]; then
realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')"
if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then
if ! grep -q "${realm_host}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts
echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}"
fi
fi
fi
fi
rm -f "${headers_file}"
}
docker_login_with_retry() {
attempts="${1:-5}"
backoff="${2:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
ensure_registry_auth_realm_host
if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
return 0
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "docker login failed; retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
retry_registry_op() {
op_name="$1"
image_ref="$2"
attempts="${3:-5}"
backoff="${4:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
if [ "${op_name}" = "push" ]; then
docker_login_with_retry 3 2
if docker push "${image_ref}"; then
return 0
fi
else
if docker pull "${image_ref}" >/dev/null; then
return 0
fi
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache"
docker_login_with_retry 5 3
retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true
BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \
--target production \
--cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-t deployable-frontend:"${HEAD_SHA}" .
bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}"
retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve frontend main digest"
exit 1
fi
echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- *failure_diagnostics_step
production-images-complete:
name: Production Images Complete
runs-on: ubuntu-act
needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
needs: [build-release-images, build-tester-images]
if: always()
timeout-minutes: 10
steps:
- name: Confirm required production/tester images are complete
run: |
set -euo pipefail
backend_main_status="${{ needs['build-backend-main-image'].result }}"
frontend_main_status="${{ needs['build-frontend-main-image'].result }}"
integration_tester_status="${{ needs['build-integration-tester-image'].result }}"
e2e_tester_status="${{ needs['build-e2e-tester-image'].result }}"
release_images_status="${{ needs['build-release-images'].result }}"
tester_images_status="${{ needs['build-tester-images'].result }}"
echo "build-backend-main-image=${backend_main_status}"
echo "build-frontend-main-image=${frontend_main_status}"
echo "build-integration-tester-image=${integration_tester_status}"
echo "build-e2e-tester-image=${e2e_tester_status}"
echo "build-release-images=${release_images_status}"
echo "build-tester-images=${tester_images_status}"
if [ "${backend_main_status}" != "success" ] || [ "${frontend_main_status}" != "success" ] || [ "${integration_tester_status}" != "success" ] || [ "${e2e_tester_status}" != "success" ]; then
if [ "${release_images_status}" != "success" ] || [ "${tester_images_status}" != "success" ]; then
echo "❌ Production Images Complete gate failed"
exit 1
fi
@@ -1678,8 +1558,8 @@ jobs:
# Run on the broader runner pool so we can still capture diagnostics when
# one of the active production image jobs fails during platform setup.
runs-on: ubuntu-act
needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
if: always() && (needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure')
needs: [build-release-images, build-tester-images]
if: always() && (needs['build-release-images'].result == 'failure' || needs['build-tester-images'].result == 'failure')
timeout-minutes: 10
steps:
- *identify_runner_step
@@ -1697,10 +1577,8 @@ jobs:
echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
echo "repository=${GITHUB_REPOSITORY:-unknown}"
echo "server_url=${GITHUB_SERVER_URL:-unknown}"
echo "build_backend_main_image_result=${{ needs['build-backend-main-image'].result }}"
echo "build_frontend_main_image_result=${{ needs['build-frontend-main-image'].result }}"
echo "build_integration_tester_image_result=${{ needs['build-integration-tester-image'].result }}"
echo "build_e2e_tester_image_result=${{ needs['build-e2e-tester-image'].result }}"
echo "build_release_images_result=${{ needs['build-release-images'].result }}"
echo "build_tester_images_result=${{ needs['build-tester-images'].result }}"
echo "=== Local Runner Telemetry (postmortem job host) ==="
uname -a || true
@@ -1757,10 +1635,8 @@ jobs:
' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \
'' \
'targets = {' \
' "Build Backend Main Image",' \
' "Build Frontend Main Image",' \
' "Build Integration Tester Image",' \
' "Build E2E Tester Image",' \
' "Build Release Images",' \
' "Build Tester Images",' \
'}' \
'' \
'print("=== targeted production image jobs ===")' \
@@ -1874,7 +1750,7 @@ jobs:
# Use the broader runner pool; only docker build steps are 8gb-exclusive.
runs-on: ubuntu-act
timeout-minutes: 20
needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image]
needs: [build_cicd, production-images-complete, build-release-images, build-tester-images]
steps:
- *identify_runner_step
@@ -1882,12 +1758,11 @@ jobs:
- name: Run runtime black-box integration checks via compose
env:
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }}
INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }}
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.integration_tester_digest_ref }}
run: |
set -euo pipefail
set -o pipefail
RUN_ID="${GITHUB_RUN_ID:-local}"
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
@@ -2179,7 +2054,7 @@ jobs:
# Use ubuntu-act runner pool for consistent availability.
runs-on: ubuntu-act
timeout-minutes: 45
needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
needs: [build_cicd, production-images-complete, build-release-images, build-tester-images]
steps:
- *identify_runner_step
@@ -2187,15 +2062,13 @@ jobs:
- name: Run E2E tests against runtime services via compose
env:
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }}
DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }}
DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_digest_ref }}
INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }}
E2E_TESTER_DIGEST_REF: ${{ needs['build-e2e-tester-image'].outputs.e2e_tester_digest_ref }}
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }}
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_tag_ref }}
DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }}
E2E_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.e2e_tester_digest_ref }}
run: |
set -euo pipefail
set -o pipefail
LOG_FILE="$(mktemp)"
ARTIFACT_DIR="$(mktemp -d)"
RUN_ID="${GITHUB_RUN_ID:-local}"

41
docs/CICD_MULTI_STAGE_BUILD.md
View File

@@ -10,7 +10,8 @@ The source of truth is `.gitea/workflows/cicd.yaml`.
Current behavior to keep in mind:
- Base image publication and CICD image publication are separate jobs in the same workflow.
- CICD base publication and complete CICD publication now run in one merged producer job.
- Release images and tester images are built in two merged producer jobs.
- Runtime image lanes (backend/frontend main + tester images) run only after source and promotion gates succeed.
- Registry operations include auth-realm host pinning and bounded retry logic for login/pull/push in critical lanes.
- Runtime integration checks consume tag and digest outputs and verify they resolve to the same immutable artifact before assertions.
@@ -123,48 +124,30 @@ Current behavior to keep in mind:
```yaml
jobs:
publish-base:
name: Build and Publish CICD Base Image
# computes base hash, checks registry, and conditionally publishes base image
build_cicd:
name: Build and Push CICD Image
needs: publish-base
# builds shared CICD image using immutable base hash
name: Build and Push CICD Images
# computes base hash, checks registry, and publishes both CICD base and complete CICD images
source-checks:
name: Source Checks
needs: build_cicd
build-backend-main-image:
needs: [build_cicd, source-checks, build-backend-base-image]
build-frontend-main-image:
needs: [build_cicd, source-checks, build-frontend-base-image]
build-integration-tester-image:
build-release-images:
needs: [build_cicd, source-checks]
build-e2e-tester-image:
build-tester-images:
needs: [build_cicd, source-checks]
production-images-complete:
needs:
[
build-backend-main-image,
build-frontend-main-image,
build-integration-tester-image,
build-e2e-tester-image,
]
needs: [build-release-images, build-tester-images]
integration-tests:
needs:
[
build_cicd,
production-images-complete,
build-backend-main-image,
build-frontend-main-image,
build-integration-tester-image,
build-release-images,
build-tester-images,
]
e2e-tests:
@@ -172,10 +155,8 @@ jobs:
[
build_cicd,
production-images-complete,
build-backend-main-image,
build-frontend-main-image,
build-integration-tester-image,
build-e2e-tester-image,
build-release-images,
build-tester-images,
]
```

5
docs/CICD_TROUBLESHOOTING_GUIDE.md
View File

@@ -87,9 +87,8 @@ Required immutable base image is not available
**Fix order**:
1. `Build and Publish CICD Base Image`
2. `Build and Push CICD Image`
3. remaining image and runtime lanes
1. `Build and Push CICD Images`
2. remaining source, image, and runtime lanes
## Performance Optimizations