diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index 43f74c4..4c31658 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -34,8 +34,8 @@ concurrency: cancel-in-progress: true jobs: - publish-base: - name: Build and Publish CICD Base Image + build_cicd: + name: Build and Push CICD Images runs-on: ubuntu-act-8gb timeout-minutes: 35 outputs: @@ -190,25 +190,6 @@ jobs: return 1 } - docker_login_with_retry() { - attempts="${1:-5}" - backoff="${2:-3}" - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - ensure_registry_auth_realm_host - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - return 0 - fi - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "docker login failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - return 1 - } - BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" @@ -223,19 +204,7 @@ jobs: ensure_registry_auth_realm_host - login_ok=false - for i in 1 2 3 4 5; do - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null; then - login_ok=true - break - fi - if [ "${i}" -lt 5 ]; then - sleep_seconds=$((3 * i)) - echo "docker login attempt ${i}/5 failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - done - if [ "${login_ok}" != "true" ]; then + if ! docker_login_with_retry 5 3 >/dev/null; then echo "❌ Failed docker login after retries" exit 1 fi @@ -365,43 +334,6 @@ jobs: retry_registry_op push "${BASE_REF_HASH}" 5 4 retry_registry_op push "${BASE_REF_LATEST}" 5 4 - - &failure_diagnostics_step - name: Failure diagnostics - if: failure() - run: | - echo "=== Failure Diagnostics ===" - date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_hostname=${HOSTNAME:-unknown}" - uname -a || true - cat /etc/os-release 2>/dev/null || true - df -h || true - free -h || true - ps aux --sort=-%mem | head -n 30 || true - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Diagnostics ===" - docker version || true - docker info || true - docker ps -a || true - docker images --digests | head -n 50 || true - fi - dmesg | tail -n 120 || true - - build_cicd: - name: Build and Push CICD Image - runs-on: ubuntu-act-8gb - needs: publish-base - timeout-minutes: 60 - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - steps: - - name: Resolve head SHA - id: meta - run: | - echo "head_sha=${{ needs['publish-base'].outputs.head_sha }}" >> "$GITHUB_OUTPUT" - - - *pre_job_prune_step - - name: Minimal checkout for CICD build inputs env: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} @@ -462,7 +394,7 @@ jobs: REGISTRY_USER: ${{ github.actor }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - BASE_HASH_FROM_BUILD: ${{ needs['publish-base'].outputs.base_hash }} + BASE_HASH_FROM_BUILD: ${{ steps.base-state.outputs.base_hash }} run: | set -euo pipefail umask 077 @@ -620,7 +552,27 @@ jobs: retry_registry_op push "${CICD_LATEST_REF}" 5 4 retry_registry_op push "${CICD_SHA_REF}" 5 4 - - *failure_diagnostics_step + - &failure_diagnostics_step + name: Failure diagnostics + if: failure() + run: | + echo "=== Failure Diagnostics ===" + date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ' + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_hostname=${HOSTNAME:-unknown}" + uname -a || true + cat /etc/os-release 2>/dev/null || true + df -h || true + free -h || true + ps aux --sort=-%mem | head -n 30 || true + if command -v docker >/dev/null 2>&1; then + echo "=== Docker Diagnostics ===" + docker version || true + docker info || true + docker ps -a || true + docker images --digests | head -n 50 || true + fi + dmesg | tail -n 120 || true build-cicd-postmortem: name: Build CICD Image Failure Postmortem @@ -831,8 +783,8 @@ jobs: - *failure_diagnostics_step - frontend-audit: - name: Frontend Dependency Audit + dependency-audits: + name: Dependency Audits (Informational) runs-on: ubuntu-act timeout-minutes: 15 continue-on-error: true @@ -862,19 +814,6 @@ jobs: echo "Proceeding: frontend audit is informational-only." fi - - *failure_diagnostics_step - - backend-audit: - name: Backend Dependency Audit - runs-on: ubuntu-act - timeout-minutes: 15 - continue-on-error: true - needs: build_cicd - steps: - - *identify_runner_step - - - *configure_registry_host_step - - *ensure_cicd_image_step - name: Run backend dependency audit env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} @@ -900,26 +839,24 @@ jobs: cicd-tests-complete: name: CICD Tests Complete runs-on: ubuntu-act - needs: [source-checks, frontend-audit, backend-audit] + needs: [source-checks, dependency-audits] if: always() steps: - name: Confirm all source lanes passed run: | set -euo pipefail source_checks_status="${{ needs['source-checks'].result }}" - frontend_audit_status="${{ needs['frontend-audit'].result }}" - backend_audit_status="${{ needs['backend-audit'].result }}" + dependency_audits_status="${{ needs['dependency-audits'].result }}" echo "source-checks=${source_checks_status}" - echo "frontend-audit=${frontend_audit_status}" - echo "backend-audit=${backend_audit_status}" + echo "dependency-audits=${dependency_audits_status}" if [ "${source_checks_status}" != "success" ]; then echo "❌ One or more CICD source lanes failed" exit 1 fi - if [ "${frontend_audit_status}" != "success" ] || [ "${backend_audit_status}" != "success" ]; then + if [ "${dependency_audits_status}" != "success" ]; then echo "⚠️ Dependency audit lane reported non-success (informational only)" fi @@ -927,13 +864,20 @@ jobs: - *failure_diagnostics_step - build-backend-base-image: - name: Build Backend Base Image + build-release-images: + name: Build Release Images runs-on: ubuntu-act-8gb needs: [build_cicd, source-checks] outputs: + head_sha: ${{ steps.meta.outputs.head_sha }} backend_base_tag_ref: ${{ steps.backend_base_ref.outputs.backend_base_tag_ref }} backend_base_digest_ref: ${{ steps.backend_base_ref.outputs.backend_base_digest_ref }} + frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} + frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} + deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} + deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} + deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} + deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} steps: - &resolve_head_sha_from_build_step name: Resolve head SHA @@ -1042,21 +986,6 @@ jobs: echo "backend_base_tag_ref=${BACKEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" echo "backend_base_digest_ref=${BACKEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - *failure_diagnostics_step - - build-frontend-base-image: - name: Build Frontend Base Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks] - outputs: - frontend_base_tag_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_tag_ref }} - frontend_base_digest_ref: ${{ steps.frontend_base_ref.outputs.frontend_base_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - name: Build and push frontend base image id: frontend_base_ref env: @@ -1151,15 +1080,212 @@ jobs: echo "frontend_base_tag_ref=${FRONTEND_BASE_TAG_REF}" >> "$GITHUB_OUTPUT" echo "frontend_base_digest_ref=${FRONTEND_BASE_DIGEST_REF}" >> "$GITHUB_OUTPUT" + - name: Build and push backend main image + id: deployable_backend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" + DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" + + docker_login_with_retry 5 3 + retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true + DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ + --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t deployable-backend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend + + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" + docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}" + + retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 + retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 + + DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve backend main digest" + exit 1 + fi + + echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + + - name: Build and push frontend main image + id: deployable_frontend_ref + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + run: | + set -euo pipefail + + ensure_registry_auth_realm_host() { + if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts + fi + + headers_file="$(mktemp)" + if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then + realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" + if [ -n "${realm_url}" ]; then + realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" + if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then + if ! grep -q "${realm_host}" /etc/hosts; then + echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts + echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" + fi + fi + fi + fi + rm -f "${headers_file}" + } + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + ensure_registry_auth_realm_host + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" + DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" + + docker_login_with_retry 5 3 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true + BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ + --target production \ + --cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t deployable-frontend:"${HEAD_SHA}" . + bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend + + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" + docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}" + + retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 + retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4 + retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 + + DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" + if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Unable to resolve frontend main digest" + exit 1 + fi + + echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" + echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" + - *failure_diagnostics_step - build-integration-tester-image: - name: Build Integration Tester Image + build-tester-images: + name: Build Tester Images runs-on: ubuntu-act-8gb needs: [build_cicd, source-checks] outputs: integration_tester_tag_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_tag_ref }} integration_tester_digest_ref: ${{ steps.integration_tester_ref.outputs.integration_tester_digest_ref }} + e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} + e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} steps: - *resolve_head_sha_from_build_step - *configure_registry_host_step @@ -1272,21 +1398,6 @@ jobs: echo "integration_tester_tag_ref=${INTEGRATION_TESTER_TAG_REF}" >> "$GITHUB_OUTPUT" echo "integration_tester_digest_ref=${INTEGRATION_TESTER_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - *failure_diagnostics_step - - build-e2e-tester-image: - name: Build E2E Tester Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks] - outputs: - e2e_tester_tag_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_tag_ref }} - e2e_tester_digest_ref: ${{ steps.e2e_tester_ref.outputs.e2e_tester_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - name: Build and push E2E tester image id: e2e_tester_ref env: @@ -1417,254 +1528,23 @@ jobs: - *failure_diagnostics_step - build-backend-main-image: - name: Build Backend Main Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks, build-backend-base-image] - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }} - deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - - name: Build and push backend main image - id: deployable_backend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - docker_login_with_retry() { - attempts="${1:-5}" - backoff="${2:-3}" - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - return 0 - fi - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "docker login failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - return 1 - } - - retry_registry_op() { - op_name="$1" - image_ref="$2" - attempts="${3:-5}" - backoff="${4:-3}" - attempt=1 - - while [ "${attempt}" -le "${attempts}" ]; do - echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" - if [ "${op_name}" = "push" ]; then - docker_login_with_retry 3 2 - if docker push "${image_ref}"; then - return 0 - fi - else - if docker pull "${image_ref}" >/dev/null; then - return 0 - fi - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - return 1 - } - - DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" - DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" - DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" - - docker_login_with_retry 5 3 - retry_registry_op pull "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 3 || true - DOCKER_BUILDKIT=1 docker build -f Dockerfile.backend \ - --cache-from "${DEPLOYABLE_BACKEND_CACHE_REF}" \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - -t deployable-backend:"${HEAD_SHA}" . - bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend - - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}" - docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_CACHE_REF}" - - retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4 - retry_registry_op push "${DEPLOYABLE_BACKEND_CACHE_REF}" 5 4 - retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3 - - DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)" - if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve backend main digest" - exit 1 - fi - - echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - - *failure_diagnostics_step - - build-frontend-main-image: - name: Build Frontend Main Image - runs-on: ubuntu-act-8gb - needs: [build_cicd, source-checks, build-frontend-base-image] - outputs: - head_sha: ${{ steps.meta.outputs.head_sha }} - deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }} - deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }} - steps: - - *resolve_head_sha_from_build_step - - *configure_registry_host_step - - *ensure_cicd_image_step - - *hydrate_build_context_from_cicd_step - - - name: Build and push frontend main image - id: deployable_frontend_ref - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ github.actor }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - run: | - set -euo pipefail - - ensure_registry_auth_realm_host() { - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - headers_file="$(mktemp)" - if curl -sSI "http://${GITEA_REGISTRY}/v2/" >"${headers_file}"; then - realm_url="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' "${headers_file}" | head -n 1)" - if [ -n "${realm_url}" ]; then - realm_host="$(printf '%s' "${realm_url}" | sed -E 's#^https?://([^/:]+).*$#\1#')" - if [ -n "${realm_host}" ] && [ "${realm_host}" != "${GITEA_REGISTRY_HOST}" ]; then - if ! grep -q "${realm_host}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${realm_host}" >> /etc/hosts - echo "Pinned registry auth realm host: ${realm_host} -> ${GITEA_REGISTRY_IP}" - fi - fi - fi - fi - rm -f "${headers_file}" - } - - docker_login_with_retry() { - attempts="${1:-5}" - backoff="${2:-3}" - attempt=1 - while [ "${attempt}" -le "${attempts}" ]; do - ensure_registry_auth_realm_host - if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - return 0 - fi - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "docker login failed; retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - return 1 - } - - retry_registry_op() { - op_name="$1" - image_ref="$2" - attempts="${3:-5}" - backoff="${4:-3}" - attempt=1 - - while [ "${attempt}" -le "${attempts}" ]; do - echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" - if [ "${op_name}" = "push" ]; then - docker_login_with_retry 3 2 - if docker push "${image_ref}"; then - return 0 - fi - else - if docker pull "${image_ref}" >/dev/null; then - return 0 - fi - fi - - if [ "${attempt}" -lt "${attempts}" ]; then - sleep_seconds=$((backoff * attempt)) - echo "Retrying in ${sleep_seconds}s" - sleep "${sleep_seconds}" - fi - attempt=$((attempt + 1)) - done - - return 1 - } - - DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" - DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" - DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" - - docker_login_with_retry 5 3 - retry_registry_op pull "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 3 || true - BUILDKIT_NO_CLIENT_TOKEN=1 DOCKER_BUILDKIT=1 docker build -f Dockerfile.frontend \ - --target production \ - --cache-from "${DEPLOYABLE_FRONTEND_CACHE_REF}" \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - -t deployable-frontend:"${HEAD_SHA}" . - bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend - - docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}" - docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_CACHE_REF}" - - retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4 - retry_registry_op push "${DEPLOYABLE_FRONTEND_CACHE_REF}" 5 4 - retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3 - - DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)" - if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then - echo "❌ Unable to resolve frontend main digest" - exit 1 - fi - - echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT" - echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT" - - - *failure_diagnostics_step - production-images-complete: name: Production Images Complete runs-on: ubuntu-act - needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + needs: [build-release-images, build-tester-images] if: always() timeout-minutes: 10 steps: - name: Confirm required production/tester images are complete run: | set -euo pipefail - backend_main_status="${{ needs['build-backend-main-image'].result }}" - frontend_main_status="${{ needs['build-frontend-main-image'].result }}" - integration_tester_status="${{ needs['build-integration-tester-image'].result }}" - e2e_tester_status="${{ needs['build-e2e-tester-image'].result }}" + release_images_status="${{ needs['build-release-images'].result }}" + tester_images_status="${{ needs['build-tester-images'].result }}" - echo "build-backend-main-image=${backend_main_status}" - echo "build-frontend-main-image=${frontend_main_status}" - echo "build-integration-tester-image=${integration_tester_status}" - echo "build-e2e-tester-image=${e2e_tester_status}" + echo "build-release-images=${release_images_status}" + echo "build-tester-images=${tester_images_status}" - if [ "${backend_main_status}" != "success" ] || [ "${frontend_main_status}" != "success" ] || [ "${integration_tester_status}" != "success" ] || [ "${e2e_tester_status}" != "success" ]; then + if [ "${release_images_status}" != "success" ] || [ "${tester_images_status}" != "success" ]; then echo "❌ Production Images Complete gate failed" exit 1 fi @@ -1678,8 +1558,8 @@ jobs: # Run on the broader runner pool so we can still capture diagnostics when # one of the active production image jobs fails during platform setup. runs-on: ubuntu-act - needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] - if: always() && (needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure') + needs: [build-release-images, build-tester-images] + if: always() && (needs['build-release-images'].result == 'failure' || needs['build-tester-images'].result == 'failure') timeout-minutes: 10 steps: - *identify_runner_step @@ -1697,10 +1577,8 @@ jobs: echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}" echo "repository=${GITHUB_REPOSITORY:-unknown}" echo "server_url=${GITHUB_SERVER_URL:-unknown}" - echo "build_backend_main_image_result=${{ needs['build-backend-main-image'].result }}" - echo "build_frontend_main_image_result=${{ needs['build-frontend-main-image'].result }}" - echo "build_integration_tester_image_result=${{ needs['build-integration-tester-image'].result }}" - echo "build_e2e_tester_image_result=${{ needs['build-e2e-tester-image'].result }}" + echo "build_release_images_result=${{ needs['build-release-images'].result }}" + echo "build_tester_images_result=${{ needs['build-tester-images'].result }}" echo "=== Local Runner Telemetry (postmortem job host) ===" uname -a || true @@ -1757,10 +1635,8 @@ jobs: ' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \ '' \ 'targets = {' \ - ' "Build Backend Main Image",' \ - ' "Build Frontend Main Image",' \ - ' "Build Integration Tester Image",' \ - ' "Build E2E Tester Image",' \ + ' "Build Release Images",' \ + ' "Build Tester Images",' \ '}' \ '' \ 'print("=== targeted production image jobs ===")' \ @@ -1874,7 +1750,7 @@ jobs: # Use the broader runner pool; only docker build steps are 8gb-exclusive. runs-on: ubuntu-act timeout-minutes: 20 - needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image] + needs: [build_cicd, production-images-complete, build-release-images, build-tester-images] steps: - *identify_runner_step @@ -1882,12 +1758,11 @@ jobs: - name: Run runtime black-box integration checks via compose env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} - INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }} + INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.integration_tester_digest_ref }} run: | set -euo pipefail - set -o pipefail RUN_ID="${GITHUB_RUN_ID:-local}" RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}" @@ -2179,7 +2054,7 @@ jobs: # Use ubuntu-act runner pool for consistent availability. runs-on: ubuntu-act timeout-minutes: 45 - needs: [build_cicd, production-images-complete, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image] + needs: [build_cicd, production-images-complete, build-release-images, build-tester-images] steps: - *identify_runner_step @@ -2187,15 +2062,13 @@ jobs: - name: Run E2E tests against runtime services via compose env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} - DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_tag_ref }} - DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-backend-main-image'].outputs.deployable_backend_digest_ref }} - DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_tag_ref }} - DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-frontend-main-image'].outputs.deployable_frontend_digest_ref }} - INTEGRATION_TESTER_DIGEST_REF: ${{ needs['build-integration-tester-image'].outputs.integration_tester_digest_ref }} - E2E_TESTER_DIGEST_REF: ${{ needs['build-e2e-tester-image'].outputs.e2e_tester_digest_ref }} + DEPLOYABLE_BACKEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_backend_tag_ref }} + DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }} + DEPLOYABLE_FRONTEND_TAG_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_tag_ref }} + DEPLOYABLE_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }} + E2E_TESTER_DIGEST_REF: ${{ needs['build-tester-images'].outputs.e2e_tester_digest_ref }} run: | set -euo pipefail - set -o pipefail LOG_FILE="$(mktemp)" ARTIFACT_DIR="$(mktemp -d)" RUN_ID="${GITHUB_RUN_ID:-local}" diff --git a/docs/CICD_MULTI_STAGE_BUILD.md b/docs/CICD_MULTI_STAGE_BUILD.md index b1f2503..8ef8f5b 100644 --- a/docs/CICD_MULTI_STAGE_BUILD.md +++ b/docs/CICD_MULTI_STAGE_BUILD.md @@ -10,7 +10,8 @@ The source of truth is `.gitea/workflows/cicd.yaml`. Current behavior to keep in mind: -- Base image publication and CICD image publication are separate jobs in the same workflow. +- CICD base publication and complete CICD publication now run in one merged producer job. +- Release images and tester images are built in two merged producer jobs. - Runtime image lanes (backend/frontend main + tester images) run only after source and promotion gates succeed. - Registry operations include auth-realm host pinning and bounded retry logic for login/pull/push in critical lanes. - Runtime integration checks consume tag and digest outputs and verify they resolve to the same immutable artifact before assertions. @@ -123,48 +124,30 @@ Current behavior to keep in mind: ```yaml jobs: - publish-base: - name: Build and Publish CICD Base Image - # computes base hash, checks registry, and conditionally publishes base image - build_cicd: - name: Build and Push CICD Image - needs: publish-base - # builds shared CICD image using immutable base hash + name: Build and Push CICD Images + # computes base hash, checks registry, and publishes both CICD base and complete CICD images source-checks: name: Source Checks needs: build_cicd - build-backend-main-image: - needs: [build_cicd, source-checks, build-backend-base-image] - - build-frontend-main-image: - needs: [build_cicd, source-checks, build-frontend-base-image] - - build-integration-tester-image: + build-release-images: needs: [build_cicd, source-checks] - build-e2e-tester-image: + build-tester-images: needs: [build_cicd, source-checks] production-images-complete: - needs: - [ - build-backend-main-image, - build-frontend-main-image, - build-integration-tester-image, - build-e2e-tester-image, - ] + needs: [build-release-images, build-tester-images] integration-tests: needs: [ build_cicd, production-images-complete, - build-backend-main-image, - build-frontend-main-image, - build-integration-tester-image, + build-release-images, + build-tester-images, ] e2e-tests: @@ -172,10 +155,8 @@ jobs: [ build_cicd, production-images-complete, - build-backend-main-image, - build-frontend-main-image, - build-integration-tester-image, - build-e2e-tester-image, + build-release-images, + build-tester-images, ] ``` diff --git a/docs/CICD_TROUBLESHOOTING_GUIDE.md b/docs/CICD_TROUBLESHOOTING_GUIDE.md index 9a60e4d..dfc1cf0 100644 --- a/docs/CICD_TROUBLESHOOTING_GUIDE.md +++ b/docs/CICD_TROUBLESHOOTING_GUIDE.md @@ -87,9 +87,8 @@ Required immutable base image is not available **Fix order**: -1. `Build and Publish CICD Base Image` -2. `Build and Push CICD Image` -3. remaining image and runtime lanes +1. `Build and Push CICD Images` +2. remaining source, image, and runtime lanes ## Performance Optimizations