diff --git a/.gitea/workflows/cicd.yaml b/.gitea/workflows/cicd.yaml index ae86ff5..a62c8c7 100644 --- a/.gitea/workflows/cicd.yaml +++ b/.gitea/workflows/cicd.yaml @@ -789,7 +789,6 @@ jobs: name: Dependency Audits (Informational) runs-on: ubuntu-act timeout-minutes: 15 - continue-on-error: true needs: build_cicd steps: - *identify_runner_step @@ -797,6 +796,7 @@ jobs: - *configure_registry_host_step - *ensure_cicd_image_step - name: Run frontend dependency audit + continue-on-error: true env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} run: | @@ -818,6 +818,7 @@ jobs: - name: Run backend dependency audit if: always() + continue-on-error: true env: HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} run: | @@ -1141,7 +1142,7 @@ jobs: return 1 } - DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend-staging" DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}" DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache" @@ -1249,7 +1250,7 @@ jobs: return 1 } - DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend-staging" DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}" DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache" @@ -2455,3 +2456,175 @@ jobs: docker info || true fi timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true + + promote-release-images: + name: Promote Staging Images To Release + runs-on: ubuntu-act + timeout-minutes: 15 + needs: [build_cicd, build-release-images, integration-tests, e2e-tests] + outputs: + release_version: ${{ steps.promote.outputs.release_version }} + release_short_sha: ${{ steps.promote.outputs.release_short_sha }} + backend_release_version_ref: ${{ steps.promote.outputs.backend_release_version_ref }} + backend_release_build_ref: ${{ steps.promote.outputs.backend_release_build_ref }} + frontend_release_version_ref: ${{ steps.promote.outputs.frontend_release_version_ref }} + frontend_release_build_ref: ${{ steps.promote.outputs.frontend_release_build_ref }} + steps: + - *identify_runner_step + + - *configure_registry_host_step + - name: Promote staging images to release tags + id: promote + env: + PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }} + STAGING_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }} + STAGING_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }} + run: | + set -euo pipefail + umask 077 + trap 'rm -f /tmp/release_ssh_key' EXIT + + docker_login_with_retry() { + attempts="${1:-5}" + backoff="${2:-3}" + attempt=1 + while [ "${attempt}" -le "${attempts}" ]; do + if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then + return 0 + fi + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "docker login failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + return 1 + } + + retry_registry_op() { + op_name="$1" + image_ref="$2" + attempts="${3:-5}" + backoff="${4:-3}" + attempt=1 + + while [ "${attempt}" -le "${attempts}" ]; do + echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}" + if [ "${op_name}" = "push" ]; then + docker_login_with_retry 3 2 + if docker push "${image_ref}"; then + return 0 + fi + else + if docker pull "${image_ref}" >/dev/null; then + return 0 + fi + fi + + if [ "${attempt}" -lt "${attempts}" ]; then + sleep_seconds=$((backoff * attempt)) + echo "Retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + attempt=$((attempt + 1)) + done + + return 1 + } + + if [ -z "${STAGING_BACKEND_DIGEST_REF}" ] || [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then + echo "❌ Missing staging image digests from build-release-images outputs" + exit 1 + fi + + mkdir -p ~/.ssh + echo "${SSH_PRIVATE_KEY}" > /tmp/release_ssh_key + chmod 600 /tmp/release_ssh_key + ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null || true + GIT_SSH_COMMAND="ssh -i /tmp/release_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" + + SHORT_SHA="$(printf '%s' "${HEAD_SHA}" | cut -c1-7)" + + HEAD_SEMVER_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \ + | awk -v sha="${HEAD_SHA}" '$1==sha {sub("refs/tags/", "", $2); print $2}' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | tail -n 1)" + + if [ -n "${HEAD_SEMVER_TAG}" ]; then + RELEASE_VERSION="${HEAD_SEMVER_TAG}" + else + BASE_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \ + | awk '{sub("refs/tags/", "", $2); print $2}' \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | tail -n 1)" + + if [ -z "${BASE_TAG}" ]; then + BASE_TAG="v0.0.0" + fi + + BASE_MAJOR="$(printf '%s' "${BASE_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')" + BASE_MINOR="$(printf '%s' "${BASE_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')" + BASE_PATCH="$(printf '%s' "${BASE_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\3/')" + + EXISTING_TAGS_JSON="$({ curl -fsSL -u "${REGISTRY_USER}:${PACKAGE_ACCESS_TOKEN}" "http://${GITEA_REGISTRY}/v2/darkhelm.org/deployable-backend/tags/list" 2>/dev/null || true; })" + + MAX_PATCH_FOR_LINE="${BASE_PATCH}" + if [ -n "${EXISTING_TAGS_JSON}" ] && command -v python3 >/dev/null 2>&1; then + REGISTRY_MAX_PATCH="$(printf '%s' "${EXISTING_TAGS_JSON}" | python3 -c "import json,re,sys; data=json.load(sys.stdin); tags=data.get('tags') or []; major=int(sys.argv[1]); minor=int(sys.argv[2]); patches=[int(m.group(3)) for tag in tags for m in [re.match(r'^v(\\d+)\\.(\\d+)\\.(\\d+)$', str(tag))] if m and int(m.group(1))==major and int(m.group(2))==minor]; print(max(patches) if patches else -1)" "${BASE_MAJOR}" "${BASE_MINOR}")" + + if [ "${REGISTRY_MAX_PATCH}" -ge "${MAX_PATCH_FOR_LINE}" ]; then + MAX_PATCH_FOR_LINE="${REGISTRY_MAX_PATCH}" + fi + fi + + NEXT_PATCH="$((MAX_PATCH_FOR_LINE + 1))" + RELEASE_VERSION="v${BASE_MAJOR}.${BASE_MINOR}.${NEXT_PATCH}" + fi + + RELEASE_BUILD_TAG="${RELEASE_VERSION}-${SHORT_SHA}" + + RELEASE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend" + RELEASE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend" + + BACKEND_RELEASE_VERSION_REF="${RELEASE_BACKEND_REPO}:${RELEASE_VERSION}" + BACKEND_RELEASE_BUILD_REF="${RELEASE_BACKEND_REPO}:${RELEASE_BUILD_TAG}" + BACKEND_RELEASE_LATEST_REF="${RELEASE_BACKEND_REPO}:latest" + FRONTEND_RELEASE_VERSION_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_VERSION}" + FRONTEND_RELEASE_BUILD_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_BUILD_TAG}" + FRONTEND_RELEASE_LATEST_REF="${RELEASE_FRONTEND_REPO}:latest" + + docker_login_with_retry 5 3 + retry_registry_op pull "${STAGING_BACKEND_DIGEST_REF}" 5 3 + retry_registry_op pull "${STAGING_FRONTEND_DIGEST_REF}" 5 3 + + BACKEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_BACKEND_DIGEST_REF}")" + FRONTEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_FRONTEND_DIGEST_REF}")" + + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_VERSION_REF}" + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_BUILD_REF}" + docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_LATEST_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_VERSION_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_BUILD_REF}" + docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_LATEST_REF}" + + retry_registry_op push "${BACKEND_RELEASE_VERSION_REF}" 5 4 + retry_registry_op push "${BACKEND_RELEASE_BUILD_REF}" 5 4 + retry_registry_op push "${BACKEND_RELEASE_LATEST_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_VERSION_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_BUILD_REF}" 5 4 + retry_registry_op push "${FRONTEND_RELEASE_LATEST_REF}" 5 4 + + echo "release_version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + echo "release_short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "backend_release_version_ref=${BACKEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT" + echo "backend_release_build_ref=${BACKEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_release_version_ref=${FRONTEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT" + echo "frontend_release_build_ref=${FRONTEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT" + + - *failure_diagnostics_step diff --git a/docs/CICD_MULTI_STAGE_BUILD.md b/docs/CICD_MULTI_STAGE_BUILD.md index 8ef8f5b..f946dd6 100644 --- a/docs/CICD_MULTI_STAGE_BUILD.md +++ b/docs/CICD_MULTI_STAGE_BUILD.md @@ -11,8 +11,11 @@ The source of truth is `.gitea/workflows/cicd.yaml`. Current behavior to keep in mind: - CICD base publication and complete CICD publication now run in one merged producer job. -- Release images and tester images are built in two merged producer jobs. -- Runtime image lanes (backend/frontend main + tester images) run only after source and promotion gates succeed. +- Dependency audits run in one informational lane; frontend and backend audits always execute and are non-blocking. +- Release and tester images are built in two merged producer jobs. +- Deployable backend/frontend runtime images are first published as staging artifacts. +- A dedicated promotion lane publishes release image tags only after integration and E2E lanes succeed. +- Release promotion uses semver tags with a short commit SHA build suffix. - Registry operations include auth-realm host pinning and bounded retry logic for login/pull/push in critical lanes. - Runtime integration checks consume tag and digest outputs and verify they resolve to the same immutable artifact before assertions. @@ -132,8 +135,14 @@ jobs: name: Source Checks needs: build_cicd + dependency-audits: + name: Dependency Audits (Informational) + needs: build_cicd + # frontend/backend audit steps are continue-on-error and backend uses if: always() + build-release-images: needs: [build_cicd, source-checks] + # publishes deployable-backend-staging and deployable-frontend-staging refs build-tester-images: needs: [build_cicd, source-checks] @@ -158,8 +167,21 @@ jobs: build-release-images, build-tester-images, ] + + promote-release-images: + needs: [build_cicd, build-release-images, integration-tests, e2e-tests] + # retags staging images to deployable-backend/deployable-frontend with: + # - release version tag (semver) + # - release build tag (-<7-char-short-sha>) + # - latest ``` +### Release Tagging Model + +- If the current commit has a semver git tag (`v..`), promotion uses that exact version. +- Otherwise, promotion finds the latest semver baseline and auto-increments patch for the next successful release in that major/minor line. +- Each promoted release also publishes a build-distinguishing tag: `-`, where `` is the 7-character commit shorthand. + ### Responsibility Split - `.gitea/workflows/cicd.yaml` owns the full CI pipeline: base image publish, diff --git a/docs/CICD_SUCCESS_SUMMARY.md b/docs/CICD_SUCCESS_SUMMARY.md index a9decc3..c01a56c 100644 --- a/docs/CICD_SUCCESS_SUMMARY.md +++ b/docs/CICD_SUCCESS_SUMMARY.md @@ -69,6 +69,24 @@ ## 🏗️ **Architecture Validation** +### **Current Release Lifecycle (2026-07)** + +- Runtime deployable images are published first to staging repositories: + - `deployable-backend-staging` + - `deployable-frontend-staging` +- Runtime validation (`Runtime Black-Box Integration Tests` and `End-to-End Tests`) must pass before release tagging. +- `Promote Release Images` retags validated staging artifacts to release repositories: + - `deployable-backend` + - `deployable-frontend` +- Published release tags: + - `latest` + - `` + - `-<7-char-short-sha>` +- Version selection rule: + - If HEAD has a semver git tag (`vX.Y.Z`), use it. + - Otherwise, auto-increment patch from the latest semver baseline for that major/minor line. +- Dependency audits are informational: frontend and backend audit steps both run and do not fail the full workflow. + ### **Working Component Integration** All major components now work seamlessly together: diff --git a/docs/CICD_TROUBLESHOOTING_GUIDE.md b/docs/CICD_TROUBLESHOOTING_GUIDE.md index dfc1cf0..6a850e5 100644 --- a/docs/CICD_TROUBLESHOOTING_GUIDE.md +++ b/docs/CICD_TROUBLESHOOTING_GUIDE.md @@ -16,6 +16,28 @@ When this guide conflicts with older examples, prefer: ## High-Value Failure Signatures (Current) +### 0. Dependency audit step fails but workflow stays green + +**Symptom**: + +```text +frontend audit reported vulnerabilities +backend audit reported vulnerabilities +``` + +and overall workflow still succeeds. + +**Cause**: expected behavior. `Dependency Audits (Informational)` is intentionally non-blocking and runs both frontend and backend audit steps. + +**Fast check**: + +1. Confirm `Dependency Audits (Informational)` ran. +2. Confirm both audit step logs are present. + +**Fix**: + +No CI fix required unless policy changes. Treat findings as remediation backlog items. + ### 1. `docker_login_with_retry: command not found` **Symptom**: @@ -90,6 +112,44 @@ Required immutable base image is not available 1. `Build and Push CICD Images` 2. remaining source, image, and runtime lanes +### 5. Release tags missing after tests passed + +**Symptom**: + +```text +staging images exist but deployable-backend/deployable-frontend tags not updated +``` + +**Cause**: promotion lane did not run or failed (`Promote Release Images`). + +**Fast check**: + +1. Confirm `Runtime Black-Box Integration Tests` and `End-to-End Tests` succeeded. +2. Check `Promote Release Images` logs for registry login, pull/tag/push failures. +3. Verify staging refs (`deployable-backend-staging`, `deployable-frontend-staging`) were emitted by `Build Release Images`. + +**Fix**: + +Re-run `Promote Release Images` after correcting registry/auth issues. + +### 6. Unexpected release version chosen + +**Symptom**: + +```text +release_version differs from expected manual guess +``` + +**Cause**: promotion versioning follows workflow rules: + +1. If HEAD commit has semver tag `vX.Y.Z`, use that exact version. +2. Otherwise, find latest semver baseline and auto-increment patch. +3. Also publish `-<7-char-short-sha>`. + +**Fix**: + +If you need an exact release version, tag the commit with semver before running promotion. + ## Performance Optimizations ### 1. Dependency-First Build Pattern diff --git a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md index b9caea0..237b122 100644 --- a/docs/DEPLOYABLE_RUNTIME_CONTRACT.md +++ b/docs/DEPLOYABLE_RUNTIME_CONTRACT.md @@ -11,6 +11,14 @@ automation work under epic #66. ## Scope +Lifecycle note: + +- CI first builds deployable runtime artifacts in staging repositories + (`deployable-backend-staging`, `deployable-frontend-staging`). +- After integration and E2E validation pass, CI promotes those immutable + artifacts to release repositories (`deployable-backend`, + `deployable-frontend`) via tag promotion. + Included: - Backend deployable runtime image requirements. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 5427940..ac801fb 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -210,7 +210,7 @@ git push 1. Push your feature branch to the remote repository 2. Navigate to the Gitea web interface 3. Create a Pull Request from your feature branch to `main` -4. Ensure all CI checks pass (100% green required) +4. Ensure required CI checks pass (informational audit warnings do not block merge) 5. Request review from team members 6. Merge only after approval and passing CI @@ -406,16 +406,16 @@ Current dispatch inputs: The pipeline is intentionally staged so expensive image jobs run only after source checks and required gates pass: -1. Publish base image lane: - `Build and Publish CICD Base Image` computes base hash, checks registry, and conditionally builds/pushes immutable + latest tags. -2. CICD image lane: - `Build and Push CICD Image` consumes base hash and publishes the shared CICD image. -3. Source lanes: - `Source Checks`, `Frontend Dependency Audit`, `Backend Dependency Audit`, then `CICD Tests Complete` gate. -4. Runtime image lanes: - backend base/frontend base, backend main/frontend main, integration tester, e2e tester, then `Production Images Complete` gate. -5. Runtime validation lanes: - `Runtime Black-Box Integration Tests` and `End-to-End Tests`. +1. Producer lane: + `Build and Push CICD Images` computes base hash, checks registry, and publishes both CICD base and complete CICD images. +2. Source + audit lanes: + `Source Checks` and `Dependency Audits (Informational)` run after producer completion. Audit failures are intentionally non-blocking so both frontend and backend audits always report. +3. Runtime image lanes: + `Build Release Images` publishes `deployable-backend-staging` and `deployable-frontend-staging`; `Build Tester Images` publishes integration/e2e tester images; then `Production Images Complete` gates downstream runtime tests. +4. Runtime validation lanes: + `Runtime Black-Box Integration Tests` and `End-to-End Tests` validate staged runtime artifacts. +5. Promotion lane: + `Promote Release Images` retags validated staging artifacts to release repos (`deployable-backend`, `deployable-frontend`) with `latest`, ``, and `-<7-char-short-sha>` tags. 6. Postmortem lanes: targeted postmortem jobs run when key lanes fail to capture diagnostics even when primary jobs fail early. @@ -446,10 +446,10 @@ Use workflow dispatch when you need deterministic reruns on a specific commit: Recommended rerun order during flaky infrastructure incidents: -1. `Build and Publish CICD Base Image` -2. `Build and Push CICD Image` -3. failing runtime image lane (`Build Integration Tester Image`, `Build Frontend Main Image`, etc.) -4. downstream integration/e2e lanes +1. `Build and Push CICD Images` +2. failing runtime image lane (`Build Release Images` or `Build Tester Images`) +3. downstream integration/e2e lanes +4. `Promote Release Images` (if staging validation succeeded but promotion failed) ### Local CI/CD Testing @@ -487,7 +487,7 @@ Build and test CI/CD images locally: 1. Navigate to your pull request in Gitea 2. Check the "Checks" tab for detailed results 3. Click on individual job names to see logs -4. All jobs must pass (100% green) before merging +4. All required jobs must pass before merging (informational dependency-audit findings are non-blocking) ## Branch Protection and Merge Requirements @@ -496,7 +496,7 @@ Build and test CI/CD images locally: The `main` branch is protected with the following requirements: 1. **No Direct Pushes**: All changes must come through pull requests -2. **CI Must Pass**: All CI/CD jobs must be 100% green +2. **CI Must Pass**: All required CI/CD jobs must pass (dependency audits are informational) 3. **Review Required**: At least one team member approval needed 4. **Up-to-date Branch**: Feature branch must be current with main @@ -507,7 +507,7 @@ The `main` branch is protected with the following requirements: 3. Address any CI failures by pushing fixes to the feature branch 4. Request and receive code review approval 5. Ensure branch is up-to-date with main -6. Merge pull request (only available when all requirements met) +6. Merge pull request (available when required jobs pass; informational audit failures do not block merge) ### If CI Fails