ci/docs: finalize staging promotion flow and docs sync

.gitea/workflows/cicd.yaml

@@ -789,7 +789,6 @@ jobs:
     name: Dependency Audits (Informational)
     runs-on: ubuntu-act
     timeout-minutes: 15
-    continue-on-error: true
     needs: build_cicd
     steps:
       - *identify_runner_step
@@ -797,6 +796,7 @@ jobs:
       - *configure_registry_host_step
       - *ensure_cicd_image_step
       - name: Run frontend dependency audit
+        continue-on-error: true
         env:
           HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
         run: |
@@ -818,6 +818,7 @@ jobs:
 
       - name: Run backend dependency audit
         if: always()
+        continue-on-error: true
         env:
           HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
         run: |
@@ -1141,7 +1142,7 @@ jobs:
             return 1
           }
 
-          DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
+          DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend-staging"
           DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
           DEPLOYABLE_BACKEND_CACHE_REF="${DEPLOYABLE_BACKEND_REPO}:cache"
 
@@ -1249,7 +1250,7 @@ jobs:
             return 1
           }
 
-          DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
+          DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend-staging"
           DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
           DEPLOYABLE_FRONTEND_CACHE_REF="${DEPLOYABLE_FRONTEND_REPO}:cache"
 
@@ -2455,3 +2456,175 @@ jobs:
             docker info || true
           fi
           timeout 10 curl -fsSIL "http://${GITEA_REGISTRY}/v2/" || true
+
+  promote-release-images:
+    name: Promote Staging Images To Release
+    runs-on: ubuntu-act
+    timeout-minutes: 15
+    needs: [build_cicd, build-release-images, integration-tests, e2e-tests]
+    outputs:
+      release_version: ${{ steps.promote.outputs.release_version }}
+      release_short_sha: ${{ steps.promote.outputs.release_short_sha }}
+      backend_release_version_ref: ${{ steps.promote.outputs.backend_release_version_ref }}
+      backend_release_build_ref: ${{ steps.promote.outputs.backend_release_build_ref }}
+      frontend_release_version_ref: ${{ steps.promote.outputs.frontend_release_version_ref }}
+      frontend_release_build_ref: ${{ steps.promote.outputs.frontend_release_build_ref }}
+    steps:
+      - *identify_runner_step
+
+      - *configure_registry_host_step
+      - name: Promote staging images to release tags
+        id: promote
+        env:
+          PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+          HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha }}
+          STAGING_BACKEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_backend_digest_ref }}
+          STAGING_FRONTEND_DIGEST_REF: ${{ needs['build-release-images'].outputs.deployable_frontend_digest_ref }}
+        run: |
+          set -euo pipefail
+          umask 077
+          trap 'rm -f /tmp/release_ssh_key' EXIT
+
+          docker_login_with_retry() {
+            attempts="${1:-5}"
+            backoff="${2:-3}"
+            attempt=1
+            while [ "${attempt}" -le "${attempts}" ]; do
+              if echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
+                return 0
+              fi
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "docker login failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+            return 1
+          }
+
+          retry_registry_op() {
+            op_name="$1"
+            image_ref="$2"
+            attempts="${3:-5}"
+            backoff="${4:-3}"
+            attempt=1
+
+            while [ "${attempt}" -le "${attempts}" ]; do
+              echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
+              if [ "${op_name}" = "push" ]; then
+                docker_login_with_retry 3 2
+                if docker push "${image_ref}"; then
+                  return 0
+                fi
+              else
+                if docker pull "${image_ref}" >/dev/null; then
+                  return 0
+                fi
+              fi
+
+              if [ "${attempt}" -lt "${attempts}" ]; then
+                sleep_seconds=$((backoff * attempt))
+                echo "Retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+              attempt=$((attempt + 1))
+            done
+
+            return 1
+          }
+
+          if [ -z "${STAGING_BACKEND_DIGEST_REF}" ] || [ -z "${STAGING_FRONTEND_DIGEST_REF}" ]; then
+            echo "❌ Missing staging image digests from build-release-images outputs"
+            exit 1
+          fi
+
+          mkdir -p ~/.ssh
+          echo "${SSH_PRIVATE_KEY}" > /tmp/release_ssh_key
+          chmod 600 /tmp/release_ssh_key
+          ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null || true
+          GIT_SSH_COMMAND="ssh -i /tmp/release_ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
+
+          SHORT_SHA="$(printf '%s' "${HEAD_SHA}" | cut -c1-7)"
+
+          HEAD_SEMVER_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
+            | awk -v sha="${HEAD_SHA}" '$1==sha {sub("refs/tags/", "", $2); print $2}' \
+            | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
+            | sort -V \
+            | tail -n 1)"
+
+          if [ -n "${HEAD_SEMVER_TAG}" ]; then
+            RELEASE_VERSION="${HEAD_SEMVER_TAG}"
+          else
+            BASE_TAG="$({ GIT_SSH_COMMAND="${GIT_SSH_COMMAND}" git ls-remote --tags --refs "${GITEA_REPO_SSH_URL}" 2>/dev/null || true; } \
+              | awk '{sub("refs/tags/", "", $2); print $2}' \
+              | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
+              | sort -V \
+              | tail -n 1)"
+
+            if [ -z "${BASE_TAG}" ]; then
+              BASE_TAG="v0.0.0"
+            fi
+
+            BASE_MAJOR="$(printf '%s' "${BASE_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\1/')"
+            BASE_MINOR="$(printf '%s' "${BASE_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\2/')"
+            BASE_PATCH="$(printf '%s' "${BASE_TAG}" | sed -E 's/^v([0-9]+)\.([0-9]+)\.([0-9]+)$/\3/')"
+
+            EXISTING_TAGS_JSON="$({ curl -fsSL -u "${REGISTRY_USER}:${PACKAGE_ACCESS_TOKEN}" "http://${GITEA_REGISTRY}/v2/darkhelm.org/deployable-backend/tags/list" 2>/dev/null || true; })"
+
+            MAX_PATCH_FOR_LINE="${BASE_PATCH}"
+            if [ -n "${EXISTING_TAGS_JSON}" ] && command -v python3 >/dev/null 2>&1; then
+              REGISTRY_MAX_PATCH="$(printf '%s' "${EXISTING_TAGS_JSON}" | python3 -c "import json,re,sys; data=json.load(sys.stdin); tags=data.get('tags') or []; major=int(sys.argv[1]); minor=int(sys.argv[2]); patches=[int(m.group(3)) for tag in tags for m in [re.match(r'^v(\\d+)\\.(\\d+)\\.(\\d+)$', str(tag))] if m and int(m.group(1))==major and int(m.group(2))==minor]; print(max(patches) if patches else -1)" "${BASE_MAJOR}" "${BASE_MINOR}")"
+
+              if [ "${REGISTRY_MAX_PATCH}" -ge "${MAX_PATCH_FOR_LINE}" ]; then
+                MAX_PATCH_FOR_LINE="${REGISTRY_MAX_PATCH}"
+              fi
+            fi
+
+            NEXT_PATCH="$((MAX_PATCH_FOR_LINE + 1))"
+            RELEASE_VERSION="v${BASE_MAJOR}.${BASE_MINOR}.${NEXT_PATCH}"
+          fi
+
+          RELEASE_BUILD_TAG="${RELEASE_VERSION}-${SHORT_SHA}"
+
+          RELEASE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
+          RELEASE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
+
+          BACKEND_RELEASE_VERSION_REF="${RELEASE_BACKEND_REPO}:${RELEASE_VERSION}"
+          BACKEND_RELEASE_BUILD_REF="${RELEASE_BACKEND_REPO}:${RELEASE_BUILD_TAG}"
+          BACKEND_RELEASE_LATEST_REF="${RELEASE_BACKEND_REPO}:latest"
+          FRONTEND_RELEASE_VERSION_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_VERSION}"
+          FRONTEND_RELEASE_BUILD_REF="${RELEASE_FRONTEND_REPO}:${RELEASE_BUILD_TAG}"
+          FRONTEND_RELEASE_LATEST_REF="${RELEASE_FRONTEND_REPO}:latest"
+
+          docker_login_with_retry 5 3
+          retry_registry_op pull "${STAGING_BACKEND_DIGEST_REF}" 5 3
+          retry_registry_op pull "${STAGING_FRONTEND_DIGEST_REF}" 5 3
+
+          BACKEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_BACKEND_DIGEST_REF}")"
+          FRONTEND_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${STAGING_FRONTEND_DIGEST_REF}")"
+
+          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_VERSION_REF}"
+          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_BUILD_REF}"
+          docker tag "${BACKEND_IMAGE_ID}" "${BACKEND_RELEASE_LATEST_REF}"
+          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_VERSION_REF}"
+          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_BUILD_REF}"
+          docker tag "${FRONTEND_IMAGE_ID}" "${FRONTEND_RELEASE_LATEST_REF}"
+
+          retry_registry_op push "${BACKEND_RELEASE_VERSION_REF}" 5 4
+          retry_registry_op push "${BACKEND_RELEASE_BUILD_REF}" 5 4
+          retry_registry_op push "${BACKEND_RELEASE_LATEST_REF}" 5 4
+          retry_registry_op push "${FRONTEND_RELEASE_VERSION_REF}" 5 4
+          retry_registry_op push "${FRONTEND_RELEASE_BUILD_REF}" 5 4
+          retry_registry_op push "${FRONTEND_RELEASE_LATEST_REF}" 5 4
+
+          echo "release_version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
+          echo "release_short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          echo "backend_release_version_ref=${BACKEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT"
+          echo "backend_release_build_ref=${BACKEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT"
+          echo "frontend_release_version_ref=${FRONTEND_RELEASE_VERSION_REF}" >> "$GITHUB_OUTPUT"
+          echo "frontend_release_build_ref=${FRONTEND_RELEASE_BUILD_REF}" >> "$GITHUB_OUTPUT"
+
+      - *failure_diagnostics_step

docs/CICD_MULTI_STAGE_BUILD.md

@@ -11,8 +11,11 @@ The source of truth is `.gitea/workflows/cicd.yaml`.
 Current behavior to keep in mind:
 
 - CICD base publication and complete CICD publication now run in one merged producer job.
-- Release images and tester images are built in two merged producer jobs.
-- Runtime image lanes (backend/frontend main + tester images) run only after source and promotion gates succeed.
+- Dependency audits run in one informational lane; frontend and backend audits always execute and are non-blocking.
+- Release and tester images are built in two merged producer jobs.
+- Deployable backend/frontend runtime images are first published as staging artifacts.
+- A dedicated promotion lane publishes release image tags only after integration and E2E lanes succeed.
+- Release promotion uses semver tags with a short commit SHA build suffix.
 - Registry operations include auth-realm host pinning and bounded retry logic for login/pull/push in critical lanes.
 - Runtime integration checks consume tag and digest outputs and verify they resolve to the same immutable artifact before assertions.
 
@@ -132,8 +135,14 @@ jobs:
     name: Source Checks
     needs: build_cicd
 
+  dependency-audits:
+    name: Dependency Audits (Informational)
+    needs: build_cicd
+    # frontend/backend audit steps are continue-on-error and backend uses if: always()
+
   build-release-images:
     needs: [build_cicd, source-checks]
+    # publishes deployable-backend-staging and deployable-frontend-staging refs
 
   build-tester-images:
     needs: [build_cicd, source-checks]
@@ -158,8 +167,21 @@ jobs:
         build-release-images,
         build-tester-images,
       ]
+
+  promote-release-images:
+    needs: [build_cicd, build-release-images, integration-tests, e2e-tests]
+    # retags staging images to deployable-backend/deployable-frontend with:
+    # - release version tag (semver)
+    # - release build tag (<semver>-<7-char-short-sha>)
+    # - latest
 ```
 
+### Release Tagging Model
+
+- If the current commit has a semver git tag (`v<major>.<minor>.<patch>`), promotion uses that exact version.
+- Otherwise, promotion finds the latest semver baseline and auto-increments patch for the next successful release in that major/minor line.
+- Each promoted release also publishes a build-distinguishing tag: `<semver>-<short_sha>`, where `<short_sha>` is the 7-character commit shorthand.
+
 ### Responsibility Split
 
 - `.gitea/workflows/cicd.yaml` owns the full CI pipeline: base image publish,

docs/CICD_SUCCESS_SUMMARY.md

@@ -69,6 +69,24 @@
 
 ## 🏗️ **Architecture Validation**
 
+### **Current Release Lifecycle (2026-07)**
+
+- Runtime deployable images are published first to staging repositories:
+  - `deployable-backend-staging`
+  - `deployable-frontend-staging`
+- Runtime validation (`Runtime Black-Box Integration Tests` and `End-to-End Tests`) must pass before release tagging.
+- `Promote Release Images` retags validated staging artifacts to release repositories:
+  - `deployable-backend`
+  - `deployable-frontend`
+- Published release tags:
+  - `latest`
+  - `<semver>`
+  - `<semver>-<7-char-short-sha>`
+- Version selection rule:
+  - If HEAD has a semver git tag (`vX.Y.Z`), use it.
+  - Otherwise, auto-increment patch from the latest semver baseline for that major/minor line.
+- Dependency audits are informational: frontend and backend audit steps both run and do not fail the full workflow.
+
 ### **Working Component Integration**
 
 All major components now work seamlessly together:

docs/CICD_TROUBLESHOOTING_GUIDE.md

@@ -16,6 +16,28 @@ When this guide conflicts with older examples, prefer:
 
 ## High-Value Failure Signatures (Current)
 
+### 0. Dependency audit step fails but workflow stays green
+
+**Symptom**:
+
+```text
+frontend audit reported vulnerabilities
+backend audit reported vulnerabilities
+```
+
+and overall workflow still succeeds.
+
+**Cause**: expected behavior. `Dependency Audits (Informational)` is intentionally non-blocking and runs both frontend and backend audit steps.
+
+**Fast check**:
+
+1. Confirm `Dependency Audits (Informational)` ran.
+2. Confirm both audit step logs are present.
+
+**Fix**:
+
+No CI fix required unless policy changes. Treat findings as remediation backlog items.
+
 ### 1. `docker_login_with_retry: command not found`
 
 **Symptom**:
@@ -90,6 +112,44 @@ Required immutable base image is not available
 1. `Build and Push CICD Images`
 2. remaining source, image, and runtime lanes
 
+### 5. Release tags missing after tests passed
+
+**Symptom**:
+
+```text
+staging images exist but deployable-backend/deployable-frontend tags not updated
+```
+
+**Cause**: promotion lane did not run or failed (`Promote Release Images`).
+
+**Fast check**:
+
+1. Confirm `Runtime Black-Box Integration Tests` and `End-to-End Tests` succeeded.
+2. Check `Promote Release Images` logs for registry login, pull/tag/push failures.
+3. Verify staging refs (`deployable-backend-staging`, `deployable-frontend-staging`) were emitted by `Build Release Images`.
+
+**Fix**:
+
+Re-run `Promote Release Images` after correcting registry/auth issues.
+
+### 6. Unexpected release version chosen
+
+**Symptom**:
+
+```text
+release_version differs from expected manual guess
+```
+
+**Cause**: promotion versioning follows workflow rules:
+
+1. If HEAD commit has semver tag `vX.Y.Z`, use that exact version.
+2. Otherwise, find latest semver baseline and auto-increment patch.
+3. Also publish `<semver>-<7-char-short-sha>`.
+
+**Fix**:
+
+If you need an exact release version, tag the commit with semver before running promotion.
+
 ## Performance Optimizations
 
 ### 1. Dependency-First Build Pattern

docs/DEPLOYABLE_RUNTIME_CONTRACT.md

@@ -11,6 +11,14 @@ automation work under epic #66.
 
 ## Scope
 
+Lifecycle note:
+
+- CI first builds deployable runtime artifacts in staging repositories
+  (`deployable-backend-staging`, `deployable-frontend-staging`).
+- After integration and E2E validation pass, CI promotes those immutable
+  artifacts to release repositories (`deployable-backend`,
+  `deployable-frontend`) via tag promotion.
+
 Included:
 
 - Backend deployable runtime image requirements.

docs/DEVELOPMENT.md

@@ -210,7 +210,7 @@ git push
 1. Push your feature branch to the remote repository
 2. Navigate to the Gitea web interface
 3. Create a Pull Request from your feature branch to `main`
-4. Ensure all CI checks pass (100% green required)
+4. Ensure required CI checks pass (informational audit warnings do not block merge)
 5. Request review from team members
 6. Merge only after approval and passing CI
 
@@ -406,16 +406,16 @@ Current dispatch inputs:
 
 The pipeline is intentionally staged so expensive image jobs run only after source checks and required gates pass:
 
-1. Publish base image lane:
-   `Build and Publish CICD Base Image` computes base hash, checks registry, and conditionally builds/pushes immutable + latest tags.
-2. CICD image lane:
-   `Build and Push CICD Image` consumes base hash and publishes the shared CICD image.
-3. Source lanes:
-   `Source Checks`, `Frontend Dependency Audit`, `Backend Dependency Audit`, then `CICD Tests Complete` gate.
-4. Runtime image lanes:
-   backend base/frontend base, backend main/frontend main, integration tester, e2e tester, then `Production Images Complete` gate.
-5. Runtime validation lanes:
-   `Runtime Black-Box Integration Tests` and `End-to-End Tests`.
+1. Producer lane:
+   `Build and Push CICD Images` computes base hash, checks registry, and publishes both CICD base and complete CICD images.
+2. Source + audit lanes:
+   `Source Checks` and `Dependency Audits (Informational)` run after producer completion. Audit failures are intentionally non-blocking so both frontend and backend audits always report.
+3. Runtime image lanes:
+   `Build Release Images` publishes `deployable-backend-staging` and `deployable-frontend-staging`; `Build Tester Images` publishes integration/e2e tester images; then `Production Images Complete` gates downstream runtime tests.
+4. Runtime validation lanes:
+   `Runtime Black-Box Integration Tests` and `End-to-End Tests` validate staged runtime artifacts.
+5. Promotion lane:
+   `Promote Release Images` retags validated staging artifacts to release repos (`deployable-backend`, `deployable-frontend`) with `latest`, `<semver>`, and `<semver>-<7-char-short-sha>` tags.
 6. Postmortem lanes:
    targeted postmortem jobs run when key lanes fail to capture diagnostics even when primary jobs fail early.
 
@@ -446,10 +446,10 @@ Use workflow dispatch when you need deterministic reruns on a specific commit:
 
 Recommended rerun order during flaky infrastructure incidents:
 
-1. `Build and Publish CICD Base Image`
-2. `Build and Push CICD Image`
-3. failing runtime image lane (`Build Integration Tester Image`, `Build Frontend Main Image`, etc.)
-4. downstream integration/e2e lanes
+1. `Build and Push CICD Images`
+2. failing runtime image lane (`Build Release Images` or `Build Tester Images`)
+3. downstream integration/e2e lanes
+4. `Promote Release Images` (if staging validation succeeded but promotion failed)
 
 ### Local CI/CD Testing
 
@@ -487,7 +487,7 @@ Build and test CI/CD images locally:
 1. Navigate to your pull request in Gitea
 2. Check the "Checks" tab for detailed results
 3. Click on individual job names to see logs
-4. All jobs must pass (100% green) before merging
+4. All required jobs must pass before merging (informational dependency-audit findings are non-blocking)
 
 ## Branch Protection and Merge Requirements
 
@@ -496,7 +496,7 @@ Build and test CI/CD images locally:
 The `main` branch is protected with the following requirements:
 
 1. **No Direct Pushes**: All changes must come through pull requests
-2. **CI Must Pass**: All CI/CD jobs must be 100% green
+2. **CI Must Pass**: All required CI/CD jobs must pass (dependency audits are informational)
 3. **Review Required**: At least one team member approval needed
 4. **Up-to-date Branch**: Feature branch must be current with main
 
@@ -507,7 +507,7 @@ The `main` branch is protected with the following requirements:
 3. Address any CI failures by pushing fixes to the feature branch
 4. Request and receive code review approval
 5. Ensure branch is up-to-date with main
-6. Merge pull request (only available when all requirements met)
+6. Merge pull request (available when required jobs pass; informational audit failures do not block merge)
 
 ### If CI Fails