diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 43f74c4..935febd 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -22,6 +22,8 @@ env: GITEA_REGISTRY: kankali.darkhelm.lan:3001 GITEA_REGISTRY_IP: 10.18.75.2 GITEA_REGISTRY_HOST: kankali.darkhelm.lan + GITEA_REGISTRY_AUTH_HOST: kankali.darkhelm.lan + GITEA_REGISTRY_AUTH_FALLBACK_HOST: dogar.darkhelm.org CI_PRUNE_AT_JOB_START: "true" CI_PRUNE_THRESHOLD_PERCENT: "90" @@ -34,8 +36,8 @@ concurrency: cancel-in-progress: true jobs: - publish-base: - name: Build and Publish CICD Base Image + build_cicd: + name: Build and Push CICD Images runs-on: ubuntu-act-8gb timeout-minutes: 35 outputs: @@ -155,12 +157,19 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi + if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts + fi + headers_file="$(mktemp)" if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" if [ -n "${realm_url}" ]; then realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then + echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability." + fi if ! grep -q "${realm_host}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" @@ -190,25 +199,6 @@ jobs: return 1 } - docker_login_with_retry() { - attempts="${1:-5}" - backoff="${2:-3}" - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - ensure_registry_auth_realm_host - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - return 0 - fi - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "docker login failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - return 1 - } - BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" @@ -223,19 +213,7 @@ jobs: ensure_registry_auth_realm_host - login_ok=false - for i in 1 2 3 4 5; do - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null; then - login_ok=true - break - fi - if [ "${i}" -lt 5 ]; then - sleep_seconds=$((3 * i)) - echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - done - if [ "${login_ok}" != "true" ]; then + if ! docker_login_with_retry 5 3 >/dev/null; then echo "❌ Failed docker login after retries" exit 1 fi @@ -298,12 +276,19 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi + if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts + fi + headers_file="$(mktemp)" if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" if [ -n "${realm_url}" ]; then realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then + echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability." + fi if ! grep -q "${realm_host}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" @@ -365,43 +350,6 @@ jobs: retry_registry_op push "${BASE_REF_HASH}" 5 4 retry_registry_op push "${BASE_REF_LATEST}" 5 4 - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - fi - dmesg | tail -n 120 || true - - build_cicd: - name: Build and Push CICD Image - runs-on: ubuntu-act-8gb - needs: publish-base - timeout-minutes: 60 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT" - - - *pre_job_prune_step - - name: Minimal checkout for CICD build inputs env: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} @@ -444,8 +392,10 @@ jobs: chmod 600 ~/.ssh/id_rsa retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null" - retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ - git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + if [ ! -d .git ]; then + retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ + git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" . + fi if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then @@ -462,7 +412,7 @@ jobs: REGISTRY_USER: ${{ github.actor }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }} + BASE_HASH_FROM_BUILD: ${{ steps.base-state.outputs.base_hash }} run: | set -euo pipefail umask 077 @@ -519,12 +469,19 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi + if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts + fi + headers_file="$(mktemp)" if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" if [ -n "${realm_url}" ]; then realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then + echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability." + fi if ! grep -q "${realm_host}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" @@ -620,7 +577,27 @@ jobs: retry_registry_op push "${CICD_LATEST_REF}" 5 4 retry_registry_op push "${CICD_SHA_REF}" 5 4 - - *failure_diagnostics_step + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + fi + dmesg | tail -n 120 || true build-cicd-postmortem: name: Build CICD Image Failure Postmortem @@ -661,12 +638,19 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi + if [ -n "${GITEA_REGISTRY_AUTH_FALLBACK_HOST:-}" ] && ! grep -q "${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_AUTH_FALLBACK_HOST}" >> /etc/hosts + fi + headers_file="$(mktemp)" if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" if [ -n "${realm_url}" ]; then realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if [ -n "${GITEA_REGISTRY_AUTH_HOST:-}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_AUTH_HOST}" ]; then + echo "Registry advertised auth realm host '${realm_host}' (expected '${GITEA_REGISTRY_AUTH_HOST}'). Applying host pin for reliability." + fi if ! grep -q "${realm_host}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" @@ -831,11 +815,11 @@ jobs: - *failure_diagnostics_step - frontend-audit: - name: Frontend Dependency Audit + dependency-audits: + name: Dependency Audits (Informational) runs-on: ubuntu-act - timeout-minutes: 15 continue-on-error: true + timeout-minutes: 15 needs: build_cicd steps: - *identify_runner_step @@ -843,6 +827,7 @@ jobs: - *configure_registry_host_step - *ensure_cicd_image_step - name: Run frontend dependency audit + continue-on-error: true env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} run: | @@ -862,20 +847,9 @@ jobs: echo "Proceeding: frontend audit is informational-only." fi - - *failure_diagnostics_step - - backend-audit: - name: Backend Dependency Audit - runs-on: ubuntu-act - timeout-minutes: 15 - continue-on-error: true - needs: build_cicd - steps: - - *identify_runner_step - - - *configure_registry_host_step - - *ensure_cicd_image_step - name: Run backend dependency audit + if: always() + continue-on-error: true env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} run: | @@ -900,26 +874,24 @@ jobs: cicd-tests-complete: name: CICD Tests Complete runs-on: ubuntu-act - needs: [source-checks, frontend-audit, backend-audit] + needs: [source-checks, dependency-audits] if: always() steps: - name: Confirm all source lanes passed run: | set -euo pipefail source_checks_status="${{ needs['source-checks'].result }}" - frontend_audit_status="${{ needs['frontend-audit'].result }}" - backend_audit_status="${{ needs['backend-audit'].result }}" + dependency_audits_status="${{ needs['dependency-audits'].result }}" echo "source-checks=${source_checks_status}" - echo "frontend-audit=${frontend_audit_status}" - echo "backend-audit=${backend_audit_status}" + echo "dependency-audits=${dependency_audits_status}" if [ "${source_checks_status}" != "success" ]; then echo "❌ One or more CICD source lanes failed" exit 1 fi - if [ "${frontend_audit_status}" != "success" ] || [ "${backend_audit_status}" != "success" ]; then + if [ "${dependency_audits_status}" != "success" ]; then echo "⚠️ Dependency audit lane reported non-success (informational only)" fi @@ -927,13 +899,20 @@ jobs: - *failure_diagnostics_step - build-backend-base-image: - name: Build Backend Base Image + build-release-images: + name: Build Release Images runs-on: ubuntu-act-8gb needs: [build_cicd, source-checks] outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }} backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }} + frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} + frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} + deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} + deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} steps: - &resolve_head_sha_from_build_step name: Resolve head SHA @@ -1042,21 +1021,6 @@ jobs: echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - *failure_diagnostics_step - - build-frontend-base-image: - name: Build Frontend Base Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks] - outputs: - frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} - frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - name: Build and push frontend base image id: frontend_base_ref env: @@ -1151,15 +1115,212 @@ jobs: echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" + - name: Build and push backend main image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend-staging" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" + + docker_login_with_retry 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true + DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ + --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t deployable-backend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend + + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}" + + retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 + retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend main digest" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - name: Build and push frontend main image + id: deployable_frontend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend-staging" + DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" + DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" + + docker_login_with_retry 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true + BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ + --target production \ + --cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t deployable-frontend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend + + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}" + + retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 + retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + + DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend main digest" + exit 1 + fi + + echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + - *failure_diagnostics_step - build-integration-tester-image: - name: Build Integration Tester Image + build-tester-images: + name: Build Tester Images runs-on: ubuntu-act-8gb needs: [build_cicd, source-checks] outputs: integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} + e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} + e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} steps: - *resolve_head_sha_from_build_step - *configure_registry_host_step @@ -1272,21 +1433,6 @@ jobs: echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - *failure_diagnostics_step - - build-e2e-tester-image: - name: Build E2E Tester Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks] - outputs: - e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} - e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - name: Build and push E2E tester image id: e2e_tester_ref env: @@ -1417,254 +1563,23 @@ jobs: - *failure_diagnostics_step - build-backend-main-image: - name: Build Backend Main Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks, build-backend-base-image] - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - - name: Build and push backend main image - id: deployable_backend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - docker_login_with_retry() { - attempts="${1:-5}" - backoff="${2:-3}" - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - return 0 - fi - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "docker login failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - return 1 - } - - retry_registry_op() { - op_name="$1" - image_ref="$2" - attempts="${3:-5}" - backoff="${4:-3}" - attempt=1 - - while [ "${attempt}" -le "${attempts}" ]; do - echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" - if [ "${op_name}" = "push" ]; then - docker_login_with_retry 3 2 - if docker push "${image_ref}"; then - return 0 - fi - else - if docker pull "${image_ref}" >/dev/null; then - return 0 - fi - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - return 1 - } - - DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" - DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" - DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" - - docker_login_with_retry 5 3 - retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true - DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ - --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - -t deployable-backend:"${HEAD_SHA}" . - bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend - - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}" - - retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 - retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4 - retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 - - DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" - if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve backend main digest" - exit 1 - fi - - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - - *failure_diagnostics_step - - build-frontend-main-image: - name: Build Frontend Main Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks, build-frontend-base-image] - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} - deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - - name: Build and push frontend main image - id: deployable_frontend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - ensure_registry_auth_realm_host() { - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - headers_file="$(mktemp)" - if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then - realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" - if [ -n "${realm_url}" ]; then - realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" - if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then - if ! grep -q "${realm_host}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts - echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" - fi - fi - fi - fi - rm -f "${headers_file}" - } - - docker_login_with_retry() { - attempts="${1:-5}" - backoff="${2:-3}" - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - ensure_registry_auth_realm_host - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - return 0 - fi - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "docker login failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - return 1 - } - - retry_registry_op() { - op_name="$1" - image_ref="$2" - attempts="${3:-5}" - backoff="${4:-3}" - attempt=1 - - while [ "${attempt}" -le "${attempts}" ]; do - echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" - if [ "${op_name}" = "push" ]; then - docker_login_with_retry 3 2 - if docker push "${image_ref}"; then - return 0 - fi - else - if docker pull "${image_ref}" >/dev/null; then - return 0 - fi - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - return 1 - } - - DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" - DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" - DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" - - docker_login_with_retry 5 3 - retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true - BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ - --target production \ - --cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - -t deployable-frontend:"${HEAD_SHA}" . - bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend - - docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" - docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}" - - retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 - retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4 - retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 - - DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" - if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve frontend main digest" - exit 1 - fi - - echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - - *failure_diagnostics_step - production-images-complete: name: Production Images Complete runs-on: ubuntu-act - needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + needs: [build-release-images, build-tester-images] if: always() timeout-minutes: 10 steps: - name: Confirm required production/tester images are complete run: | set -euo pipefail - backend_main_status="${{ needs['build-backend-main-image'].result }}" - frontend_main_status="${{ needs['build-frontend-main-image'].result }}" - integration_tester_status="${{ needs['build-integration-tester-image'].result }}" - e2e_tester_status="${{ needs['build-e2e-tester-image'].result }}" + release_images_status="${{ needs['build-release-images'].result }}" + tester_images_status="${{ needs['build-tester-images'].result }}" - echo "build-backend-main-image=${backend_main_status}" - echo "build-frontend-main-image=${frontend_main_status}" - echo "build-integration-tester-image=${integration_tester_status}" - echo "build-e2e-tester-image=${e2e_tester_status}" + echo "build-release-images=${release_images_status}" + echo "build-tester-images=${tester_images_status}" - if [ "${backend_main_status}" != "success" ] || [ "${frontend_main_status}" != "success" ] || [ "${integration_tester_status}" != "success" ] || [ "${e2e_tester_status}" != "success" ]; then + if [ "${release_images_status}" != "success" ] || [ "${tester_images_status}" != "success" ]; then echo "❌ Production Images Complete gate failed" exit 1 fi @@ -1678,8 +1593,8 @@ jobs: # Run on the broader runner pool so we can still capture diagnostics when # one of the active production image jobs fails during platform setup. runs-on: ubuntu-act - needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] - if: always() && (needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') + needs: [build-release-images, build-tester-images] + if: always() && (needs['build-release-images'].result == 'failure' || needs['build-tester-images'].result == 'failure') timeout-minutes: 10 steps: - *identify_runner_step @@ -1697,10 +1612,8 @@ jobs: echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" echo "repository=${GITHUB_REPOSITORY:-unknown}" echo "server_url=${GITHUB_SERVER_URL:-unknown}" - echo "build_backend_main_image_result=${{ needs['build-backend-main-image'].result }}" - echo "build_frontend_main_image_result=${{ needs['build-frontend-main-image'].result }}" - echo "build_integration_tester_image_result=${{ needs['build-integration-tester-image'].result }}" - echo "build_e2e_tester_image_result=${{ needs['build-e2e-tester-image'].result }}" + echo "build_release_images_result=${{ needs['build-release-images'].result }}" + echo "build_tester_images_result=${{ needs['build-tester-images'].result }}" echo "=== Local Runner Telemetry (postmortem job host) ===" uname -a || true @@ -1757,10 +1670,8 @@ jobs: ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \ '' \ 'targets = {' \ - ' "Build Backend Main Image",' \ - ' "Build Frontend Main Image",' \ - ' "Build Integration Tester Image",' \ - ' "Build E2E Tester Image",' \ + ' "Build Release Images",' \ + ' "Build Tester Images",' \ '}' \ '' \ 'print("=== targeted production image jobs ===")' \ @@ -1874,7 +1785,7 @@ jobs: # Use the broader runner pool; only docker build steps are 8gb-exclusive. runs-on: ubuntu-act timeout-minutes: 20 - needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image] + needs: [build_cicd, production-images-complete, build-release-images, build-tester-images] steps: - *identify_runner_step @@ -1882,12 +1793,11 @@ jobs: - name: Run runtime black-box integration checks via compose env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} - INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.integration_tester_digest_ref }} run: | set -euo pipefail - set -o pipefail RUN_ID="${GITHUB_RUN_ID:-local}" RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" @@ -2179,7 +2089,7 @@ jobs: # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act timeout-minutes: 45 - needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + needs: [build_cicd, production-images-complete, build-release-images, build-tester-images] steps: - *identify_runner_step @@ -2187,15 +2097,13 @@ jobs: - name: Run E2E tests against runtime services via compose env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} - DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }} - DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_digest_ref }} - INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} - E2E_TESTER_DIGEST_REF: ${{ needs['build-e2e-tester-image'].outputs.e2e_tester_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }} + DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_tag_ref }} + DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }} + E2E_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.e2e_tester_digest_ref }} run: | set -euo pipefail - set -o pipefail LOG_FILE="$(mktemp)" ARTIFACT_DIR="$(mktemp -d)" RUN_ID="${GITHUB_RUN_ID:-local}" @@ -2579,3 +2487,270 @@ jobs: docker info || true fi timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true + + promote-release-images: + name: Promote Staging Images To Release + runs-on: ubuntu-act + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + timeout-minutes: 15 + needs: [build_cicd, build-release-images, integration-tests, e2e-tests] + outputs: + release_version: ${{ steps.promote.outputs.release_version }} + release_short_sha: ${{ steps.promote.outputs.release_short_sha }} + release_tagged_version: ${{ steps.promote.outputs.release_tagged_version }} + previous_release_version: ${{ steps.promote.outputs.previous_release_version }} + backend_release_version_ref: ${{ steps.promote.outputs.backend_release_version_ref }} + backend_release_tagged_ref: ${{ steps.promote.outputs.backend_release_tagged_ref }} + backend_release_build_ref: ${{ steps.promote.outputs.backend_release_build_ref }} + frontend_release_version_ref: ${{ steps.promote.outputs.frontend_release_version_ref }} + frontend_release_tagged_ref: ${{ steps.promote.outputs.frontend_release_tagged_ref }} + frontend_release_build_ref: ${{ steps.promote.outputs.frontend_release_build_ref }} + steps: + - *identify_runner_step + + - *configure_registry_host_step + - name: Promote staging images to release tags + id: promote + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + STAGING_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }} + STAGING_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }} + run: | + set -euo pipefail + umask 077 + TMP_REPO_DIR="$(mktemp -d)" + trap 'rm -f /tmp/release_ssh_key; rm -rf "${TMP_REPO_DIR}"' EXIT + + retry_cmd() { + attempts="${1:-5}" + backoff="${2:-2}" + shift 2 + + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if "$@"; then + return 0 + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Command failed (attempt ${attempt}/${attempts}): $*" + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + echo "Command failed after ${attempts} attempts: $*" + return 1 + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + if [ -z "${STAGING_BACKEND_DIGEST_REF}" ] || [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Missing staging image digests from build-release-images outputs" + exit 1 + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > /tmp/release_ssh_key + chmod 600 /tmp/release_ssh_key + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null || true + GIT_SSH_COMMAND="ssh -i /tmp/release_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" + + SHORT_SHA="$(printf '%s' "${HEAD_SHA}" | cut -c1-7)" + + HEAD_SEMVER_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \ + | awk -v sha="${HEAD_SHA}" '$1==sha {sub("refs/tags/", "", $2); print $2}' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | tail -n 1)" + + if [ -z "${HEAD_SEMVER_TAG}" ]; then + ALL_SEMVER_TAGS="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \ + | awk '{sub("refs/tags/", "", $2); print $2}' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V)" + + LAST_SEMVER_TAG="$(printf '%s\n' "${ALL_SEMVER_TAGS}" | tail -n 1)" + if [ -z "${LAST_SEMVER_TAG}" ]; then + LAST_SEMVER_TAG="v0.0.0" + fi + + TAG_MAJOR="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')" + TAG_MINOR="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')" + TAG_PATCH="$(printf '%s' "${LAST_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\3/')" + NEXT_PATCH="$((TAG_PATCH + 1))" + HEAD_SEMVER_TAG="v${TAG_MAJOR}.${TAG_MINOR}.${NEXT_PATCH}" + + retry_cmd 5 2 git -C "${TMP_REPO_DIR}" init -q + retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" remote add origin "${GITEA_REPO_SSH_URL}" + retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --no-tags --depth 200 origin "${HEAD_SHA}" + git -C "${TMP_REPO_DIR}" config user.name "gitea-actions" + git -C "${TMP_REPO_DIR}" config user.email "gitea-actions@darkhelm.local" + git -C "${TMP_REPO_DIR}" tag "${HEAD_SEMVER_TAG}" "${HEAD_SHA}" + retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" push origin "refs/tags/${HEAD_SEMVER_TAG}" + echo "Created patch release tag ${HEAD_SEMVER_TAG} for ${HEAD_SHA}" + fi + + TAG_MAJOR="$(printf '%s' "${HEAD_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')" + TAG_MINOR="$(printf '%s' "${HEAD_SEMVER_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')" + RELEASE_VERSION="v${TAG_MAJOR}.${TAG_MINOR}.0" + RELEASE_TAGGED_VERSION="${HEAD_SEMVER_TAG}" + RELEASE_BUILD_TAG="${RELEASE_TAGGED_VERSION}-${SHORT_SHA}" + + RELEASE_TAG_LIST="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \ + | awk '{sub("refs/tags/", "", $2); print $2}' \ + | grep -E '^v[0-9]+\.[0-9]+\.0$' \ + | sort -V)" + + PREVIOUS_RELEASE_VERSION="$(printf '%s\n' "${RELEASE_TAG_LIST}" | awk -v current="${RELEASE_VERSION}" 'NF {if ($0 == current) {print prev; found=1; exit} prev=$0} END {if (!found) print prev}')" + + RELEASE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + RELEASE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + + BACKEND_RELEASE_VERSION_REF="${RELEASE_BACKEND_REPO}:${RELEASE_VERSION}" + BACKEND_RELEASE_TAGGED_REF="${RELEASE_BACKEND_REPO}:${RELEASE_TAGGED_VERSION}" + BACKEND_RELEASE_BUILD_REF="${RELEASE_BACKEND_REPO}:${RELEASE_BUILD_TAG}" + BACKEND_RELEASE_LATEST_REF="${RELEASE_BACKEND_REPO}:latest" + FRONTEND_RELEASE_VERSION_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_VERSION}" + FRONTEND_RELEASE_TAGGED_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_TAGGED_VERSION}" + FRONTEND_RELEASE_BUILD_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_BUILD_TAG}" + FRONTEND_RELEASE_LATEST_REF="${RELEASE_FRONTEND_REPO}:latest" + + if [ ! -d "${TMP_REPO_DIR}/.git" ]; then + retry_cmd 5 2 git -C "${TMP_REPO_DIR}" init -q + retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" remote add origin "${GITEA_REPO_SSH_URL}" + retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --no-tags --depth 200 origin "${HEAD_SHA}" + fi + retry_cmd 5 3 env GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git -C "${TMP_REPO_DIR}" fetch --tags --force origin + + RELEASE_NOTES_FILE="$(mktemp)" + PREVIOUS_RELEASE_GIT_REF="${PREVIOUS_RELEASE_VERSION}" + if [ -z "${PREVIOUS_RELEASE_GIT_REF}" ]; then + PREVIOUS_RELEASE_VERSION="v0.0.0" + fi + + if [ -n "${PREVIOUS_RELEASE_GIT_REF}" ]; then + LOG_RANGE="${PREVIOUS_RELEASE_VERSION}..${HEAD_SHA}" + CHANGES_HEADER="Changes since previous release ${PREVIOUS_RELEASE_VERSION}:" + else + LOG_RANGE="${HEAD_SHA}" + CHANGES_HEADER="Changes since baseline ${PREVIOUS_RELEASE_VERSION}:" + fi + + { + echo "## Release ${RELEASE_VERSION}" + echo + echo "- Release series tag: ${RELEASE_VERSION}" + echo "- Version tag on commit: ${RELEASE_TAGGED_VERSION}" + echo "- Build tag: ${RELEASE_BUILD_TAG}" + if [ -n "${PREVIOUS_RELEASE_VERSION}" ]; then + echo "- Previous release: ${PREVIOUS_RELEASE_VERSION}" + else + echo "- Previous release: (none)" + fi + echo + echo "${CHANGES_HEADER}" + git -C "${TMP_REPO_DIR}" log --no-merges --pretty='- %s (%h)' "${LOG_RANGE}" || true + } > "${RELEASE_NOTES_FILE}" + + if ! grep -q '^- ' "${RELEASE_NOTES_FILE}"; then + { + echo + echo "- No commit-summary entries were found in the selected range." + } >> "${RELEASE_NOTES_FILE}" + fi + + { + echo "### Generated Release Notes" + cat "${RELEASE_NOTES_FILE}" + } >> "$GITHUB_STEP_SUMMARY" + + docker_login_with_retry 5 3 + retry_registry_op pull "${STAGING_BACKEND_DIGEST_REF}" 5 3 + retry_registry_op pull "${STAGING_FRONTEND_DIGEST_REF}" 5 3 + + BACKEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_BACKEND_DIGEST_REF}")" + FRONTEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_FRONTEND_DIGEST_REF}")" + + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_VERSION_REF}" + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_TAGGED_REF}" + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_BUILD_REF}" + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_LATEST_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_VERSION_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_TAGGED_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_BUILD_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_LATEST_REF}" + + retry_registry_op push "${BACKEND_RELEASE_VERSION_REF}" 5 4 + retry_registry_op push "${BACKEND_RELEASE_TAGGED_REF}" 5 4 + retry_registry_op push "${BACKEND_RELEASE_BUILD_REF}" 5 4 + retry_registry_op push "${BACKEND_RELEASE_LATEST_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_VERSION_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_TAGGED_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_BUILD_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_LATEST_REF}" 5 4 + + echo "release_version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + echo "release_short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "release_tagged_version=${RELEASE_TAGGED_VERSION}" >> "$GITHUB_OUTPUT" + echo "previous_release_version=${PREVIOUS_RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + echo "backend_release_version_ref=${BACKEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT" + echo "backend_release_tagged_ref=${BACKEND_RELEASE_TAGGED_REF}" >> "$GITHUB_OUTPUT" + echo "backend_release_build_ref=${BACKEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_release_version_ref=${FRONTEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_release_tagged_ref=${FRONTEND_RELEASE_TAGGED_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_release_build_ref=${FRONTEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step diff --git a/docs/CICD_MULTI_STAGE_BUILD.md b/docs/CICD_MULTI_STAGE_BUILD.md index 6092969..b1c6db5 100644 --- a/docs/CICD_MULTI_STAGE_BUILD.md +++ b/docs/CICD_MULTI_STAGE_BUILD.md @@ -4,6 +4,22 @@ This project uses a two-stage Docker build approach to optimize CI/CD performance by separating stable base dependencies from project-specific code and dependencies. +## Current Workflow Status (2026-07) + +The source of truth is `.gitea/workflows/cicd.yaml`. + +Current behavior to keep in mind: + +- CICD base publication and complete CICD publication now run in one merged producer job. +- Dependency audits run in one informational lane; frontend and backend audits always execute and are non-blocking. +- Release and tester images are built in two merged producer jobs. +- Deployable backend/frontend runtime images are first published as staging artifacts. +- A dedicated promotion lane publishes release image tags only after integration and E2E lanes succeed, and only for automated `push` runs on `main`. +- If a qualifying `main` commit is untagged, CI auto-creates the next patch tag starting from a `v0.0.0` baseline. +- Release promotion publishes release-line tags (`v..0`) plus patch/build tags (`v..`, `v..-<7-char-short-sha>`). +- Registry operations include auth-realm host pinning and bounded retry logic for login/pull/push in critical lanes. +- Runtime integration checks consume tag and digest outputs and verify they resolve to the same immutable artifact before assertions. + ## Architecture ### Stage 1: Base Image (`Dockerfile.cicd-base`) @@ -43,7 +59,10 @@ This project uses a two-stage Docker build approach to optimize CI/CD performanc - Pre-commit hook environments (leverages global pre-commit) - Project-specific tooling verification -**Registry**: `dogar.darkhelm.org/darkhelm.org/plex-playlist/cicd:latest` +**Registry**: + +- `kankali.darkhelm.lan:3001/darkhelm.org/plex-playlist-cicd:latest` +- `kankali.darkhelm.lan:3001/darkhelm.org/plex-playlist-cicd:` **Rebuild Triggers**: Every CI/CD run (contains project-specific code and dependencies) @@ -109,37 +128,64 @@ This project uses a two-stage Docker build approach to optimize CI/CD performanc ```yaml jobs: - publish-base: - name: Build and Publish CICD Base Image - steps: - - name: Compute base hash - # Uses scripts/compute-cicd-base-hash.sh - # Hashes Dockerfile.cicd-base and .dockerignore + build_cicd: + name: Build and Push CICD Images + # computes base hash, checks registry, and publishes both CICD base and complete CICD images - - name: Build and push base image - if: needs_build == 'true' - # Only runs when the immutable base tag is missing or force rebuild is requested - # Tags with both hash and 'latest' + source-checks: + name: Source Checks + needs: build_cicd - - name: Verify published base image - # Confirms the immutable tag is visible and pullable before success + dependency-audits: + name: Dependency Audits (Informational) + needs: build_cicd + # frontend/backend audit steps are continue-on-error and backend uses if: always() - prepare-base-ref: - name: Prepare CICD Base Reference - steps: - - name: Compute immutable base ref - # Uses the same helper as the base workflow + build-release-images: + needs: [build_cicd, source-checks] + # publishes deployable-backend-staging and deployable-frontend-staging refs - setup: - name: Build and Push CICD Complete Image - needs: prepare-base-ref - steps: - - name: Build and push complete CICD image - # Always runs, inherits from cicd-base: - # Waits briefly for the immutable base tag to appear, then fails clearly if missing - # Contains project code and dependencies + build-tester-images: + needs: [build_cicd, source-checks] + + production-images-complete: + needs: [build-release-images, build-tester-images] + + integration-tests: + needs: + [ + build_cicd, + production-images-complete, + build-release-images, + build-tester-images, + ] + + e2e-tests: + needs: + [ + build_cicd, + production-images-complete, + build-release-images, + build-tester-images, + ] + + promote-release-images: + needs: [build_cicd, build-release-images, integration-tests, e2e-tests] + # retags staging images to deployable-backend/deployable-frontend with: + # - release series tag (..0) + # - exact git semver tag from HEAD (authoritative release tag) + # - release build tag (-<7-char-short-sha>) + # - latest ``` +### Release Tagging Model + +- Promotion is release-tag driven: if HEAD has no semver git tag (`v..`), promotion is skipped. +- Release identity is derived from the git tag on HEAD. +- Release series tag is normalized to `..0`. +- Each promoted release also publishes a build-distinguishing tag: `-`, where `` is the 7-character commit shorthand. +- Promotion emits generated release notes summarizing commit subjects since the previous `..0` release tag. + ### Responsibility Split - `.gitea/workflows/cicd.yaml` owns the full CI pipeline: base image publish, @@ -259,6 +305,15 @@ RUN export NODE_OPTIONS="--max-old-space-size=1024" && \ - Fix: run or rerun the `CICD Base Image` workflow, or wait for it to finish when a PR changes base inputs - Design note: main CI intentionally fails instead of rebuilding the base locally +### Registry Auth Realm Timeout + +- Symptom: `Client.Timeout exceeded while awaiting headers` when docker push/pull calls token endpoint +- Cause: registry challenge realm host is not consistently reachable/resolved from runner +- Current workflow mitigation: + - host mapping for registry host + - challenge parsing and auth-realm host pinning + - bounded login and push/pull retries in image lanes + ### Common Issues - **SSH Key Problems**: Ensure SSH_PRIVATE_KEY secret is properly configured diff --git a/docs/CICD_SUCCESS_SUMMARY.md b/docs/CICD_SUCCESS_SUMMARY.md index a9decc3..cfcedac 100644 --- a/docs/CICD_SUCCESS_SUMMARY.md +++ b/docs/CICD_SUCCESS_SUMMARY.md @@ -69,6 +69,26 @@ ## 🏗️ **Architecture Validation** +### **Current Release Lifecycle (2026-07)** + +- Runtime deployable images are published first to staging repositories: + - `deployable-backend-staging` + - `deployable-frontend-staging` +- Runtime validation (`Runtime Black-Box Integration Tests` and `End-to-End Tests`) must pass before release tagging. +- `Promote Release Images` retags validated staging artifacts to release repositories, but only on automated `push` runs to `main`: + - `deployable-backend` + - `deployable-frontend` +- Published release tags: + - `latest` + - `v..0` + - `v..` + - `v..-<7-char-short-sha>` +- Version selection rule: + - If the qualifying `main` commit already has a semver git tag (`vX.Y.Z`), use it. + - Otherwise, auto-create and use the next patch tag from the latest semver tag in the repository. + - If no prior semver tag exists, bootstrap from `v0.0.0` and create `v0.0.1`. +- Dependency audits are informational: frontend and backend audit steps both run and do not fail the full workflow. + ### **Working Component Integration** All major components now work seamlessly together: diff --git a/docs/CICD_TROUBLESHOOTING_GUIDE.md b/docs/CICD_TROUBLESHOOTING_GUIDE.md index b78a796..b6b3ae1 100644 --- a/docs/CICD_TROUBLESHOOTING_GUIDE.md +++ b/docs/CICD_TROUBLESHOOTING_GUIDE.md @@ -4,6 +4,155 @@ This document captures the specific optimizations, fixes, and troubleshooting approaches developed during November 2025 for the plex-playlist CI/CD pipeline. Each entry includes the problem, root cause analysis, solution implementation, and performance impact. +## Current Workflow Reference (2026-07) + +The authoritative workflow is `.gitea/workflows/cicd.yaml`. + +When this guide conflicts with older examples, prefer: + +- current job names and dependencies in `cicd.yaml` +- current registry endpoint `kankali.darkhelm.lan:3001` +- current retry and auth-realm host pinning logic embedded in image build lanes + +## High-Value Failure Signatures (Current) + +### 0. Dependency audit step fails but workflow stays green + +**Symptom**: + +```text +frontend audit reported vulnerabilities +backend audit reported vulnerabilities +``` + +and overall workflow still succeeds. + +**Cause**: expected behavior. `Dependency Audits (Informational)` is intentionally non-blocking and runs both frontend and backend audit steps. + +**Fast check**: + +1. Confirm `Dependency Audits (Informational)` ran. +2. Confirm both audit step logs are present. + +**Fix**: + +No CI fix required unless policy changes. Treat findings as remediation backlog items. + +### 1. `docker_login_with_retry: command not found` + +**Symptom**: + +```text +line : docker_login_with_retry: command not found +``` + +**Cause**: shell helper function referenced in a job step but missing in that same step's `run` block. + +**Fast check**: + +1. Open failing job step in `.gitea/workflows/cicd.yaml`. +2. Confirm `docker_login_with_retry()` is defined before first call in that block. + +**Fix**: + +Add the helper definition locally in that step block (functions do not cross step boundaries). + +### 2. Registry token timeout while pushing/pulling + +**Symptom**: + +```text +Client.Timeout exceeded while awaiting headers +... /v2/token?...service=container_registry +``` + +**Cause**: runner resolves/pins registry host, but token realm host from `WWW-Authenticate` challenge is unresolved/unreachable. + +**Fast check**: + +1. Verify `Configure registry host resolution` step ran. +2. Confirm auth realm host pinning logic is present in failing lane. +3. Check lane-specific login/push retry helpers are active. + +**Fix**: + +Use `ensure_registry_auth_realm_host` + `docker_login_with_retry` + bounded `retry_registry_op` in the failing lane. + +### 3. Empty downstream digest/tag outputs + +**Symptom**: + +```text +evaluated to '%!t(string=)' +``` + +**Cause**: upstream image lane failed before writing expected outputs (`*_tag_ref`, `*_digest_ref`). + +**Fast check**: + +1. Inspect upstream image job conclusion (`Build Frontend Main Image`, `Build Integration Tester Image`, etc.). +2. Confirm output writes (`echo key=value >> $GITHUB_OUTPUT`) execute after push and digest resolution. + +**Fix**: + +Repair failing upstream lane first; downstream expressions become valid once outputs are emitted. + +### 4. Base image publication mismatch + +**Symptom**: + +```text +Required immutable base image is not available +``` + +**Cause**: expected hash tag not yet published or failed publication lane. + +**Fix order**: + +1. `Build and Push CICD Images` +2. remaining source, image, and runtime lanes + +### 5. Release tags missing after tests passed + +**Symptom**: + +```text +staging images exist but deployable-backend/deployable-frontend tags not updated +``` + +**Cause**: promotion lane did not run or failed (`Promote Release Images`). + +**Fast check**: + +1. Confirm `Runtime Black-Box Integration Tests` and `End-to-End Tests` succeeded. +2. Confirm the workflow run was an automated `push` on `main`; promotion is skipped for PR validation and other non-main events. +3. Check `Promote Release Images` logs for registry login, tag creation, pull/tag/push failures. +4. Verify staging refs (`deployable-backend-staging`, `deployable-frontend-staging`) were emitted by `Build Release Images`. + +**Fix**: + +Re-run `Promote Release Images` after correcting registry/auth issues. + +### 6. Unexpected release version chosen + +**Symptom**: + +```text +release_version differs from expected manual guess +``` + +**Cause**: promotion versioning follows workflow rules: + +1. Promotion only runs for automated `push` events on `main`. +2. If the `main` commit already has semver tag `vX.Y.Z`, use that exact patch tag. +3. Otherwise, find the latest semver tag in the repository and auto-create the next patch tag. +4. If no semver tag exists yet, bootstrap from `v0.0.0` and create `v0.0.1`. +5. Also publish `vX.Y.0` and `vX.Y.Z-<7-char-short-sha>`. + +**Fix**: + +If you need an exact patch version, tag the `main` commit with semver before promotion runs; otherwise let CI assign the next patch automatically. + ## Performance Optimizations ### 1. Dependency-First Build Pattern diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index b9caea0..237b122 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -11,6 +11,14 @@ automation work under epic #66. ## Scope +Lifecycle note: + +- CI first builds deployable runtime artifacts in staging repositories + (`deployable-backend-staging`, `deployable-frontend-staging`). +- After integration and E2E validation pass, CI promotes those immutable + artifacts to release repositories (`deployable-backend`, + `deployable-frontend`) via tag promotion. + Included: - Backend deployable runtime image requirements. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 82a4f75..30ff2e8 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -17,10 +17,11 @@ This document outlines how to set up your development environment and work with - **[Poe Task Reference](POE_TASK_REFERENCE.md)** - Complete guide to unified development tasks - **[CI/CD Multi-Stage Build Architecture](CICD_MULTI_STAGE_BUILD.md)** - Technical details of the optimized build system -- **[CI/CD Troubleshooting](GITEA_ACTIONS_TROUBLESHOOTING.md)** - Common issues and solutions +- **[CI/CD Troubleshooting](CICD_TROUBLESHOOTING_GUIDE.md)** - Current CI lane failures and remediation paths - **[Secure Docker CI/CD](SECURE_DOCKER_CICD.md)** - Security considerations and practices - **[Deployable Runtime Contract](DEPLOYABLE_RUNTIME_CONTRACT.md)** - Canonical backend/frontend runtime artifact contract and exclusion rules - **[ADR003: Deployable Runtime Image Contract Boundaries](adr/ADR003-deployable_runtime_image_contract.md)** - Decision record for deployable runtime boundaries +- **[ADR004: Registry Image Resolution and Auth Resilience Policy](adr/ADR004-registry-image-resolution-and-auth-resilience.md)** - Cross-workflow reliability policy for registry/auth/image fallback behavior ## Deployable Runtime Artifacts @@ -209,7 +210,7 @@ git push 1. Push your feature branch to the remote repository 2. Navigate to the Gitea web interface 3. Create a Pull Request from your feature branch to `main` -4. Ensure all CI checks pass (100% green required) +4. Ensure required CI checks pass (informational audit warnings do not block merge) 5. Request review from team members 6. Merge only after approval and passing CI @@ -388,58 +389,68 @@ pre-commit run end-of-file-fixer --all-files ### Pipeline Overview -The CI/CD pipeline uses a **multi-stage build architecture** for optimal performance: +The canonical CI workflow is `.gitea/workflows/cicd.yaml`. -- **Stage 1**: Source-level fast checks (format, lint, type-check) - **hard promotion gate** -- **Stage 2**: Build base image (system dependencies, Python, Node.js) - **cached across runs** -- **Stage 3**: Build complete image (project code and dependencies) - **rebuilt every time** +Current triggers: -Pipeline triggers: +- Push on `main` and `develop` +- Pull requests targeting `main` and `develop` +- Manual `workflow_dispatch` -- Push to any branch -- Pull requests to `main` or `develop` +Current dispatch inputs: -### Multi-Stage Build Benefits ✅ **VALIDATED SUCCESSFUL** +- `head_sha`: commit SHA to process +- `force_rebuild_base`: force base image publication -**Performance Gains**: +### Current Job Topology -- **85% build time improvement**: 3-5 minutes (down from 15-25 minutes) -- Base image cached when `Dockerfile.cicd-base` unchanged (~95% of runs) -- **100% success rate** achieved with optimized dependency management -- Raspberry Pi 4GB workers handle builds efficiently with resource optimization +The pipeline is intentionally staged so expensive image jobs run only after source checks and required gates pass: -**Architecture**: +1. Producer lane: + `Build and Push CICD Images` computes base hash, checks registry, and publishes both CICD base and complete CICD images. +2. Source + audit lanes: + `Source Checks` and `Dependency Audits (Informational)` run after producer completion. Audit failures are intentionally non-blocking so both frontend and backend audits always report. +3. Runtime image lanes: + `Build Release Images` publishes `deployable-backend-staging` and `deployable-frontend-staging`; `Build Tester Images` publishes integration/e2e tester images; then `Production Images Complete` gates downstream runtime tests. +4. Runtime validation lanes: + `Runtime Black-Box Integration Tests` and `End-to-End Tests` validate staged runtime artifacts. +5. Promotion lane: + `Promote Release Images` runs only for automated `push` events on `main`. It retags validated staging artifacts to release repos (`deployable-backend`, `deployable-frontend`) with `latest`, `v..0`, `v..`, and `v..-<7-char-short-sha>` tags. If the `main` commit is untagged, CI creates the next patch tag automatically; if no prior semver tag exists, the bootstrap baseline is `v0.0.0`. +6. Postmortem lanes: + targeted postmortem jobs run when key lanes fail to capture diagnostics even when primary jobs fail early. -- `cicd-base:latest` - System dependencies (Python 3.14, Node.js 24, build tools, pre-installed dev packages) -- `cicd:latest` - Complete environment (project code + optimized dependency installation) +### Reliability and Traceability Behavior -**Recent Optimizations** (November 2025): +Current workflow behavior includes: -- **Dependency-first build pattern** prevents cache invalidation on code changes -- **Yarn PnP state regeneration** ensures reliable frontend builds -- **Network-resilient E2E testing** with simplified Docker operations -- **Memory-optimized frontend installations** with proper swap configuration +- registry auth realm host pinning from `WWW-Authenticate` challenge when registry tokens are issued from a different host +- bounded retry logic for docker login/pull/push operations in image lanes +- digest/tag contract checks for deployable image references before runtime black-box tests +- release-note summary generation in the promotion lane (commit bullets since the previous release tag, or from the `v0.0.0` bootstrap baseline on first release) +- context hydration for image-build lanes by copying `/workspace` from the published CICD image +- runner split between `ubuntu-act` and `ubuntu-act-8gb` based on lane resource requirements -For detailed technical information, see [CI/CD Multi-Stage Build Architecture](CICD_MULTI_STAGE_BUILD.md). +For details of base/complete image build strategy, see [CI/CD Multi-Stage Build Architecture](CICD_MULTI_STAGE_BUILD.md). +For incident handling, see [CI/CD Troubleshooting](CICD_TROUBLESHOOTING_GUIDE.md). -### Pipeline Jobs +### Operator Quick Reference -All jobs run in parallel after the setup phases: +Use workflow dispatch when you need deterministic reruns on a specific commit: -1. **Source Fast Gate**: - Backend source checks (Ruff format/lint, Pyright), frontend source checks (Prettier, ESLint, TypeScript), and dispatches downstream build only on success. +```bash +# Example: rerun CI against an explicit commit +# input head_sha= -2. **Setup Base**: Builds and pushes base Docker image (conditional) -3. **Setup Complete**: Builds and pushes complete CI/CD Docker image -4. **Code Quality**: - Trailing whitespace check, end-of-file formatting, YAML syntax validation, and TOML syntax validation. +# Example: force base image republish +# input force_rebuild_base=true +``` -5. **Backend Validation**: - Ruff formatting check, Ruff linting, Pyright type checking, Darglint docstring validation, unit tests with coverage, integration tests, and doctests (xdoctest). +Recommended rerun order during flaky infrastructure incidents: -6. **Frontend Validation**: - Prettier formatting check, ESLint linting, TypeScript compilation, and unit tests with coverage. - - E2E tests (Playwright) +1. `Build and Push CICD Images` +2. failing runtime image lane (`Build Release Images` or `Build Tester Images`) +3. downstream integration/e2e lanes +4. `Promote Release Images` (if staging validation succeeded but promotion failed) ### Local CI/CD Testing @@ -477,7 +488,7 @@ Build and test CI/CD images locally: 1. Navigate to your pull request in Gitea 2. Check the "Checks" tab for detailed results 3. Click on individual job names to see logs -4. All jobs must pass (100% green) before merging +4. All required jobs must pass before merging (informational dependency-audit findings are non-blocking) ## Branch Protection and Merge Requirements @@ -486,7 +497,7 @@ Build and test CI/CD images locally: The `main` branch is protected with the following requirements: 1. **No Direct Pushes**: All changes must come through pull requests -2. **CI Must Pass**: All CI/CD jobs must be 100% green +2. **CI Must Pass**: All required CI/CD jobs must pass (dependency audits are informational) 3. **Review Required**: At least one team member approval needed 4. **Up-to-date Branch**: Feature branch must be current with main @@ -497,7 +508,7 @@ The `main` branch is protected with the following requirements: 3. Address any CI failures by pushing fixes to the feature branch 4. Request and receive code review approval 5. Ensure branch is up-to-date with main -6. Merge pull request (only available when all requirements met) +6. Merge pull request (available when required jobs pass; informational audit failures do not block merge) ### If CI Fails diff --git a/docs/RENOVATE_SETUP_GUIDE.md b/docs/RENOVATE_SETUP_GUIDE.md index 6915651..ef1b57b 100644 --- a/docs/RENOVATE_SETUP_GUIDE.md +++ b/docs/RENOVATE_SETUP_GUIDE.md @@ -4,6 +4,25 @@ Renovate is an automated dependency update tool that creates pull requests to keep your project dependencies up to date. This guide covers setting up Renovate for the plex-playlist project with optimal configuration. +## Repository Current Mode (2026-07) + +This repository runs Renovate through `.gitea/workflows/renovate.yml`. + +Current operational behavior: + +1. Uses `ubuntu-act-8gb` runner due to npm/registry memory pressure. +2. Prepares Renovate container image with mirror-first strategy: + +- primary: `kankali.darkhelm.lan:3001/darkhelm.org/renovate:41` +- fallback: `ghcr.io/renovatebot/renovate:41` + +3. Uses digest-aware freshness checks before deciding whether local cached image is current. +4. Selects endpoint dynamically between internal and external candidates based on preflight reachability. +5. Selects token via preflight checks (repo access required) with fallback order from configured secrets. +6. Uses constrained memory settings (`RENOVATE_NODE_ARGS`) and disables OSV alerts in this runner profile. + +Treat workflow behavior as source of truth; use this doc as operator guidance. + ## Setup Options ### Option 1: GitHub App (Recommended for GitHub) @@ -89,7 +108,7 @@ on: jobs: renovate: - runs-on: ubuntu-act + runs-on: ubuntu-act-8gb steps: - name: Checkout uses: actions/checkout@v4 @@ -101,7 +120,7 @@ jobs: token: ${{ secrets.RENOVATE_TOKEN }} env: RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: https://dogar.darkhelm.org/api/v1 + RENOVATE_ENDPOINT: selected at runtime from internal/external candidates ``` ## Configuration Explanation @@ -204,6 +223,37 @@ docker run --rm \ 4. **Large Updates**: Major version updates may need manual review 5. **Docker Registry**: Ensure base image updates don't break builds +### Current Workflow-Specific Failures + +1. **Renovate image pull failures** + +- Symptom: both mirror and GHCR candidates fail. +- Check: runner DNS/egress and registry auth token validity. + +2. **Endpoint preflight failures** + +- Symptom: cannot reach both internal and external API endpoints. +- Check: endpoint host mapping, TLS mode, and runner network route. + +3. **Token access failures** + +- Symptom: API `/repos//` check returns non-200. +- Check: token scopes and secret ordering. + +4. **OOM or abrupt termination** + +- Symptom: process exits under memory pressure. +- Check: `RENOVATE_NODE_ARGS`, PR concurrency limits, and optional feature toggles. + +## Operator Notes + +For this repository, prefer updating `.gitea/workflows/renovate.yml` over local one-off Renovate service changes. Keep docs and workflow in sync after changing: + +- endpoint selection logic +- token preflight/fallback order +- image source policy (mirror/fallback) +- memory and concurrency guardrails + ### Quick Validation For basic JSON validation without installing Renovate: diff --git a/docs/adr/ADR004-registry-image-resolution-and-auth-resilience.md b/docs/adr/ADR004-registry-image-resolution-and-auth-resilience.md new file mode 100644 index 0000000..399d4e9 --- /dev/null +++ b/docs/adr/ADR004-registry-image-resolution-and-auth-resilience.md @@ -0,0 +1,69 @@ +# ADR004: Registry Image Resolution and Auth Resilience Policy + +- Status: Accepted +- Date: 2026-07-16 + +## Context + +The CI and Renovate workflows run on self-hosted runners with intermittent DNS and network instability. Recent failures showed that image pull/push reliability depends on more than simple retries: + +- registry token realms may resolve to a different host than the registry endpoint +- mirror images can become stale relative to upstream +- downstream lanes require immutable references from upstream lanes +- Renovate must operate across internal/external endpoint paths with token variability + +Without an explicit policy, each job implements ad hoc behavior and drift reintroduces flakiness. + +## Decision + +Adopt a cross-workflow reliability policy for registry/image operations: + +1. Prefer mirrored images first, then fallback upstream sources when mirror resolution fails. +2. Use digest-aware freshness checks when deciding whether local image cache is current. +3. Pin registry auth realm hosts when `WWW-Authenticate` challenge host differs from registry host. +4. Use bounded login/pull/push retry wrappers in image publication and consumption lanes. +5. Propagate and verify immutable digest references for downstream runtime validation lanes. +6. Keep retry counts/timeouts configurable as operational tuning, not architectural invariants. + +## Scope + +This ADR applies to: + +- `.gitea/workflows/cicd.yaml` +- `.gitea/workflows/renovate.yml` +- helper scripts used for mirrored image resolution and lane orchestration + +This ADR does not prescribe exact retry constants or runner sizing thresholds. + +## Consequences + +Positive: + +- reduced CI flakiness from token realm host mismatch and transient registry failures +- stronger traceability via digest-first downstream checks +- clearer operator expectations for endpoint/token/image fallback behavior +- consistent reliability approach across CICD and Renovate workflows + +Negative: + +- increased workflow script complexity and duplicated helper logic inside isolated step shells +- additional maintenance burden to keep helper patterns consistent across lanes + +## Alternatives Considered + +1. Keep per-job ad hoc retries only. + +- Rejected due to repeated regressions and inconsistent behavior. + +2. Depend solely on upstream registries. + +- Rejected due to local network and availability constraints. + +3. Rebuild missing artifacts in downstream lanes. + +- Rejected because it breaks publish-once/consume-many behavior and weakens traceability. + +## Related Decisions + +- `ADR002-cicd_base_image_tagging.md` +- `ADR003-deployable_runtime_image_contract.md`