diff --git a/.gitea/workflows/cicd.yml b/.gitea/workflows/cicd.yml index bb1a4af..407ffa4 100644 --- a/.gitea/workflows/cicd.yml +++ b/.gitea/workflows/cicd.yml @@ -114,20 +114,24 @@ jobs: exit 1 fi - setup-base: - name: Build and Push CICD Base Image + prepare-base-ref: + name: Prepare CICD Base Reference runs-on: ubuntu-act-8gb needs: registry-preflight + outputs: + base_hash: ${{ steps.compute-base.outputs.base_hash }} + base_image: ${{ steps.compute-base.outputs.base_image }} + steps: - - name: Minimal checkout for base Dockerfile + - name: Minimal checkout for base hash inputs env: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} GITHUB_SHA: ${{ github.sha }} run: | umask 077 trap 'rm -f ~/.ssh/id_rsa' EXIT - echo "=== Minimal Repository Checkout for Base Dockerfile ===" + echo "=== Minimal Repository Checkout for Base Hash Inputs ===" if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts @@ -145,7 +149,7 @@ jobs: exit 1 fi - # Clone just enough to get the Dockerfile + # Clone just enough to get the base hash inputs. GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git clone --depth 1 --no-checkout \ "${GITEA_REPO_SSH_URL}" . @@ -154,165 +158,35 @@ jobs: if [ -n "${GITHUB_SHA}" ] && \ GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore - echo "✓ Checked out Dockerfile.cicd-base from commit ${GITHUB_SHA}" + git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + echo "✓ Checked out base hash inputs from commit ${GITHUB_SHA}" else - git checkout HEAD -- Dockerfile.cicd-base .dockerignore - echo "⚠ Falling back to default branch HEAD for Dockerfile checkout" + git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh + echo "⚠ Falling back to default branch HEAD for base hash inputs checkout" fi - # Clean up SSH key for security + chmod +x scripts/compute-cicd-base-hash.sh rm -f ~/.ssh/id_rsa - echo "✓ Dockerfile.cicd-base ready for build" + echo "✓ Base hash inputs ready" - - name: Check if base image needs rebuilding - id: check-base - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ secrets.REGISTRY_USER || github.actor }} + - name: Compute immutable base image reference + id: compute-base run: | - echo "=== Computing CICD Base Image Hash ===" + echo "=== Computing Immutable CICD Base Reference ===" - # Compute deterministic base hash for tagging and cache lookup. - BASE_HASH=$(sha256sum Dockerfile.cicd-base | cut -d' ' -f1 | head -c16) - BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - echo "Base Dockerfile hash: ${BASE_HASH}" - echo "base_hash=${BASE_HASH}" >> $GITHUB_OUTPUT - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - # Real check: if hash-tagged base exists in registry, skip rebuild. - # If registry/login/metadata check is unavailable, fail open and rebuild. - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null 2>&1; then - echo "⚠ Registry login failed during base check; forcing rebuild" - echo "needs_build=true" >> $GITHUB_OUTPUT - exit 0 - fi - - if timeout 30 docker manifest inspect "${BASE_REF_HASH}" >/dev/null 2>&1; then - echo "✓ Base image already exists in registry: ${BASE_REF_HASH}" - echo "needs_build=false" >> $GITHUB_OUTPUT - else - echo "ℹ Base image not found (or registry metadata unavailable): ${BASE_REF_HASH}" - echo "needs_build=true" >> $GITHUB_OUTPUT - fi - - - name: Build and push base image - if: steps.check-base.outputs.needs_build == 'true' - env: - PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }} - REGISTRY_USER: ${{ secrets.REGISTRY_USER || github.actor }} - BASE_HASH: ${{ steps.check-base.outputs.base_hash }} - run: | - echo "=== Building CICD Base Image ===" - - if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then - echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts - fi - - # Defensive fallback if output propagation failed for any reason. - BASE_HASH="${BASE_HASH:-$(sha256sum Dockerfile.cicd-base | cut -d' ' -f1 | head -c16)}" - if [ -z "${BASE_HASH}" ]; then - echo "❌ BASE_HASH is empty; refusing to tag/push invalid image references" - exit 1 - fi + BASE_HASH=$(./scripts/compute-cicd-base-hash.sh) + BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" echo "Base hash: ${BASE_HASH}" - - # Enable Docker BuildKit - export DOCKER_BUILDKIT=1 - - # Build base image (no secrets needed for base dependencies) - echo "Building base image..." - if docker build -f Dockerfile.cicd-base \ - --build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \ - -t cicd-base:latest .; then - echo "✓ Base image built successfully" - else - echo "❌ Failed to build base image" - exit 1 - fi - - BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" - ARCHIVE_PATH="/tmp/cicd-base.tar" - - # Primary push path (docker); fallback path (skopeo) if daemon registry settings are ignored. - push_ref() { - ref="$1" - - for i in 1 2 3; do - echo "Docker push attempt ${i}/3 for ${ref}..." - if docker push "${ref}"; then - echo "✓ Docker push succeeded for ${ref}" - return 0 - fi - - if [ "${i}" -lt 3 ]; then - sleep 10 - fi - done - - echo "⚠ Docker push failed for ${ref}; trying skopeo fallback" - - if ! command -v skopeo >/dev/null 2>&1; then - if command -v apt-get >/dev/null 2>&1; then - apt-get update && apt-get install -y skopeo - elif command -v apk >/dev/null 2>&1; then - apk add --no-cache skopeo - fi - fi - - if ! command -v skopeo >/dev/null 2>&1; then - echo "❌ skopeo not available for fallback push" - return 1 - fi - - # Use docker-archive source to avoid docker-daemon API version mismatches. - if [ ! -f "${ARCHIVE_PATH}" ]; then - echo "Creating local image archive for fallback push..." - if ! docker save cicd-base:latest -o "${ARCHIVE_PATH}"; then - echo "❌ Failed to create docker archive for fallback push" - return 1 - fi - fi - - if skopeo copy \ - --dest-creds "${REGISTRY_USER}:${PACKAGE_ACCESS_TOKEN}" \ - --dest-tls-verify=false \ - "docker-archive:${ARCHIVE_PATH}" \ - "docker://${ref}"; then - echo "✓ Skopeo fallback push succeeded for ${ref}" - return 0 - fi - - echo "❌ Skopeo fallback push failed for ${ref}" - return 1 - } - - # Login once for docker push path. - if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then - echo "⚠ Docker login failed; skopeo fallback will still be attempted" - fi - - # Tag and publish both immutable hash and latest. - docker tag cicd-base:latest "${BASE_REF_HASH}" - docker tag cicd-base:latest "${BASE_REF_LATEST}" - - if push_ref "${BASE_REF_HASH}" && push_ref "${BASE_REF_LATEST}"; then - echo "✓ Base image published to registry: ${BASE_REF_HASH} and latest" - else - echo "⚠ Failed to publish base image to registry; continuing with local-fallback workflow" - echo "⚠ Downstream setup job will build base image locally if registry pull is unavailable" - fi + echo "Base image: ${BASE_IMAGE}" + echo "base_hash=${BASE_HASH}" >> $GITHUB_OUTPUT + echo "base_image=${BASE_IMAGE}" >> $GITHUB_OUTPUT setup: name: Build and Push CICD Complete Image runs-on: ubuntu-act-8gb - needs: setup-base + needs: prepare-base-ref steps: - name: Minimal checkout for Dockerfile @@ -349,10 +223,10 @@ jobs: if [ -n "${GITHUB_SHA}" ] && \ GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \ git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then - git checkout FETCH_HEAD -- Dockerfile.cicd Dockerfile.cicd-base .dockerignore - echo "✓ Checked out Dockerfiles from commit ${GITHUB_SHA}" + git checkout FETCH_HEAD -- Dockerfile.cicd + echo "✓ Checked out Dockerfile.cicd from commit ${GITHUB_SHA}" else - git checkout HEAD -- Dockerfile.cicd Dockerfile.cicd-base .dockerignore + git checkout HEAD -- Dockerfile.cicd echo "⚠ Falling back to default branch HEAD for Dockerfile checkout" fi @@ -367,6 +241,7 @@ jobs: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} GITHUB_SHA: ${{ github.sha }} REGISTRY_USER: ${{ secrets.REGISTRY_USER || github.actor }} + BASE_IMAGE: ${{ needs.prepare-base-ref.outputs.base_image }} run: | umask 077 trap 'rm -f /tmp/ssh_key' EXIT @@ -379,35 +254,26 @@ jobs: # Login to registry echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - # Prefer the immutable hash-tagged base image so unchanged base Dockerfiles - # reuse the cached registry image instead of rebuilding locally. - BASE_HASH=$(sha256sum Dockerfile.cicd-base | cut -d' ' -f1 | head -c16) - BASE_IMAGE_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}" - BASE_IMAGE_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest" - BASE_IMAGE="${BASE_IMAGE_HASH}" + echo "Waiting for immutable base image: ${BASE_IMAGE}" - echo "Checking immutable base image: ${BASE_IMAGE_HASH}" + for i in 1 2 3 4 5 6; do + echo "Base availability check ${i}/6..." - if timeout 30 docker pull "${BASE_IMAGE_HASH}" 2>/dev/null; then - echo "✓ Base image pulled from immutable hash tag" - elif timeout 30 docker pull "${BASE_IMAGE_LATEST}" 2>/dev/null; then - BASE_IMAGE="${BASE_IMAGE_LATEST}" - echo "✓ Base image pulled from latest tag" - else - echo "ℹ Base image not available in registry by hash or latest" - echo "Building base image locally as fallback..." + if timeout 30 docker manifest inspect "${BASE_IMAGE}" >/dev/null 2>&1 && \ + timeout 60 docker pull "${BASE_IMAGE}" >/dev/null 2>&1; then + echo "✓ Base image pulled from immutable hash tag" + break + fi - # Check if base Dockerfile exists and build it locally - if [ -f "Dockerfile.cicd-base" ]; then - export DOCKER_BUILDKIT=1 - docker build -f Dockerfile.cicd-base -t cicd-base-local:latest . - BASE_IMAGE="cicd-base-local:latest" - echo "✓ Base image built locally: ${BASE_IMAGE}" - else - echo "❌ Cannot find Dockerfile.cicd-base for fallback build" + if [ "${i}" -eq 6 ]; then + echo "❌ Required immutable base image is not available: ${BASE_IMAGE}" + echo "Publish the base image via the CICD Base Image workflow before rerunning main CI." exit 1 fi - fi + + echo "⚠ Base image not available yet; waiting 10s for publish workflow" + sleep 10 + done # Create temporary SSH key file for BuildKit secrets echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key diff --git a/docs/CICD_MULTI_STAGE_BUILD.md b/docs/CICD_MULTI_STAGE_BUILD.md index 9e0a090..c4a2872 100644 --- a/docs/CICD_MULTI_STAGE_BUILD.md +++ b/docs/CICD_MULTI_STAGE_BUILD.md @@ -21,9 +21,11 @@ This project uses a two-stage Docker build approach to optimize CI/CD performanc - **Playwright CLI and browsers** (Chromium, Firefox, WebKit) - ~400MB cached - SSH helper scripts for git operations -**Registry**: `dogar.darkhelm.org/darkhelm.org/plex-playlist/cicd-base:latest` +**Registry**: +- Immutable: `kankali.darkhelm.lan:3001/darkhelm.org/plex-playlist-cicd-base:` +- Convenience: `kankali.darkhelm.lan:3001/darkhelm.org/plex-playlist-cicd-base:latest` -**Rebuild Triggers**: Only when `Dockerfile.cicd-base` changes (detected via SHA256 hash) +**Rebuild Triggers**: Only when `Dockerfile.cicd-base`, `.dockerignore`, or the shared hash helper changes ### Stage 2: Complete Image (`Dockerfile.cicd`) **Purpose**: Inherits from base and adds project code and dependencies. @@ -87,36 +89,50 @@ This project uses a two-stage Docker build approach to optimize CI/CD performanc ### Caching Strategy 1. **Docker Layer Caching**: Docker automatically caches unchanged layers -2. **Registry Caching**: Base image pulled from registry if available -3. **Hash-Based Invalidation**: Base image tagged with Dockerfile hash -4. **Conditional Building**: Base only rebuilds when `Dockerfile.cicd-base` changes +2. **Registry Caching**: Base image is built once and then pulled by all runners +3. **Hash-Based Invalidation**: Base image tagged with a shared helper-derived hash +4. **Conditional Building**: Base workflow rebuilds only when base inputs change ## CI/CD Workflow ```yaml jobs: - setup-base: - name: Build and Push CICD Base Image + publish-base: + name: Build and Publish CICD Base Image steps: - - name: Check if base image needs rebuilding - # Calculates SHA256 of Dockerfile.cicd-base - # Pulls existing image with hash tag if available - # Sets needs_build=false if image exists + - name: Compute base hash + # Uses scripts/compute-cicd-base-hash.sh + # Hashes Dockerfile.cicd-base and .dockerignore - name: Build and push base image if: needs_build == 'true' - # Only runs when base Dockerfile changed + # Only runs when the immutable base tag is missing or force rebuild is requested # Tags with both hash and 'latest' + - name: Verify published base image + # Confirms the immutable tag is visible and pullable before success + + prepare-base-ref: + name: Prepare CICD Base Reference + steps: + - name: Compute immutable base ref + # Uses the same helper as the base workflow + setup: name: Build and Push CICD Complete Image - needs: setup-base + needs: prepare-base-ref steps: - name: Build and push complete CICD image - # Always runs, inherits from base:latest + # Always runs, inherits from cicd-base: + # Waits briefly for the immutable base tag to appear, then fails clearly if missing # Contains project code and dependencies ``` +### Responsibility Split +- `.gitea/workflows/cicd-base.yml` owns base publication and verification. +- `.gitea/workflows/cicd.yml` owns complete-image build, tests, and deployment checks. +- Main CI never rebuilds the base image locally. + ## Local Development ### Building Base Image @@ -150,6 +166,9 @@ rm /tmp/ssh_key ./scripts/build-cicd-local.sh ``` +The local script now uses `scripts/compute-cicd-base-hash.sh`, so the local hash-tagged +base image matches the immutable tag CI expects. + ## Memory Optimization ### Raspberry Pi 4GB Constraints @@ -183,9 +202,15 @@ RUN export NODE_OPTIONS="--max-old-space-size=1024" && \ ### Troubleshooting 1. **Base Image Issues**: Check `Dockerfile.cicd-base` syntax and system dependencies 2. **Complete Image Issues**: Usually project dependency or SSH access problems -3. **Cache Misses**: Verify registry connectivity and hash calculation +3. **Cache Misses**: Verify registry connectivity and the shared base hash calculation 4. **Memory Issues**: Check swap setup and Node.js memory limits +### Missing Immutable Base Tag +- Symptom: main CI fails with `Required immutable base image is not available` +- Cause: the expected `cicd-base:` has not been published yet +- Fix: run or rerun the `CICD Base Image` workflow, or wait for it to finish when a PR changes base inputs +- Design note: main CI intentionally fails instead of rebuilding the base locally + ### Common Issues - **SSH Key Problems**: Ensure SSH_PRIVATE_KEY secret is properly configured - **Registry Authentication**: Verify PACKAGE_ACCESS_TOKEN permissions diff --git a/docs/CICD_TROUBLESHOOTING_GUIDE.md b/docs/CICD_TROUBLESHOOTING_GUIDE.md index 6f6d5ce..4a5be32 100644 --- a/docs/CICD_TROUBLESHOOTING_GUIDE.md +++ b/docs/CICD_TROUBLESHOOTING_GUIDE.md @@ -217,6 +217,31 @@ RUN echo "=== Base Image Optimization Status ===" && \ ## Troubleshooting Playbook +### Missing Immutable Base Image + +**Symptom**: +``` +❌ Required immutable base image is not available: kankali.darkhelm.lan:3001/darkhelm.org/plex-playlist-cicd-base: +Publish the base image via the CICD Base Image workflow before rerunning main CI. +``` + +**Meaning**: The main CI workflow computed the expected base hash correctly, but the +dedicated base-image workflow has not published that immutable tag yet. + +**Checks**: +1. Confirm whether `Dockerfile.cicd-base`, `.dockerignore`, or `scripts/compute-cicd-base-hash.sh` changed in the branch. +2. Check the `CICD Base Image` workflow for the same commit or PR. +3. Verify the registry contains `plex-playlist-cicd-base:`. + +**Resolution**: +1. If the base workflow is still running, rerun main CI after it completes. +2. If the base workflow did not trigger, run it manually with `force_rebuild=false`. +3. If the tag should be republished despite already existing, run it manually with `force_rebuild=true`. + +**Why CI does not self-heal here**: rebuilding the base inside the main workflow would +make different runners produce and consume different local artifacts, which defeats the +publish-once/consume-many design. + ### Docker Build Failures #### 1. rsync Command Not Found diff --git a/docs/adr/ADR002-cicd_base_image_tagging.md b/docs/adr/ADR002-cicd_base_image_tagging.md index d0fe428..dd9ebc6 100644 --- a/docs/adr/ADR002-cicd_base_image_tagging.md +++ b/docs/adr/ADR002-cicd_base_image_tagging.md @@ -6,33 +6,45 @@ ## Context CICD builds depend on a shared base image containing stable system dependencies. -To optimize build times while preserving correctness, the pipeline uses cache-aware -image reuse and fallback behavior. +To optimize build times across a fleet of self-hosted runners while preserving +correctness, the pipeline now separates base-image publication from application CI. +The expensive base image should be built once per content change, pushed to the +registry under an immutable tag, and then pulled by every runner that needs it. This branch reinforced the workflow by ensuring hash-driven base tags are always computed and non-empty before tagging/pushing, preventing invalid image references. +It also expanded the base hash inputs to include `.dockerignore` so context changes +that affect the base build invalidate the published tag. ## Decision -Adopt hash-based tagging for the CICD base image with latest fallback: +Adopt a hardened split workflow for the CICD base image: -1. Compute a deterministic hash from Dockerfile.cicd-base content. -2. Publish base image as both: +1. Compute a deterministic hash from `Dockerfile.cicd-base` and `.dockerignore`. +2. Publish the base image in a dedicated workflow as both: - hash-specific tag: cicd-base: - rolling tag: cicd-base:latest -3. Prefer pulling hash-specific image; fallback to latest when needed. -4. Validate/fallback hash propagation in workflow to avoid empty tag references. +3. Treat the hash-specific tag as the source of truth for all CI consumers. +4. Keep `latest` only as a convenience tag for humans and manual debugging. +5. Make the main CI workflow consume only the immutable hash tag and fail clearly + if that base image has not been published yet. +6. Add bounded polling in main CI to tolerate short publish/consume races between + the dedicated base workflow and the main workflow. ## Consequences Positive: - Faster CI via stable base-layer reuse - Better traceability from base image to Dockerfile content +- Single publish, many pulls across the runner fleet - Reduced risk of malformed image references in workflow execution +- Clearer separation of concerns between artifact publication and application CI Negative: - Slightly more workflow complexity - Registry stores additional hash-tagged images +- Main CI now fails fast when the expected base image is missing instead of + rebuilding it locally ## Alternatives Considered @@ -40,3 +52,6 @@ Negative: - Rejected due to reduced traceability and cache precision - Build base image every run - Rejected due to slower pipelines and wasted compute +- Allow main CI to rebuild missing base images locally + - Rejected because it reintroduces split-brain behavior across runners and + defeats publish-once/consume-many optimization diff --git a/scripts/build-cicd-local.sh b/scripts/build-cicd-local.sh index 908d134..e5fa781 100755 --- a/scripts/build-cicd-local.sh +++ b/scripts/build-cicd-local.sh @@ -16,6 +16,7 @@ NC='\033[0m' # No Color BASE_IMAGE_TAG="cicd-base:local" COMPLETE_IMAGE_TAG="cicd:local" PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BASE_HASH_HELPER="$PROJECT_DIR/scripts/compute-cicd-base-hash.sh" TEMP_SSH_KEY="/tmp/cicd_build_ssh_key" # Functions @@ -93,6 +94,11 @@ check_requirements() { exit 1 fi + if [[ ! -x "$BASE_HASH_HELPER" ]]; then + log_error "Base hash helper is missing or not executable: $BASE_HASH_HELPER" + exit 1 + fi + log_success "Requirements check passed" } @@ -107,8 +113,8 @@ build_base_image() { local start_time=$(date +%s) - # Calculate base Dockerfile hash for tagging - local base_hash=$(sha256sum "$PROJECT_DIR/Dockerfile.cicd-base" | cut -d' ' -f1 | head -c16) + # Calculate the canonical base hash used by CI and local builds. + local base_hash=$("$BASE_HASH_HELPER") log_info "Base Dockerfile hash: $base_hash" # Build base image @@ -129,7 +135,9 @@ build_base_image() { # Show image size local image_size=$(docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" | grep "$BASE_IMAGE_TAG" | awk '{print $2}') log_info "Base image size: $image_size (includes Playwright browsers)" -}build_complete_image() { +} + +build_complete_image() { log_info "Building CICD complete image..." # Check if base image exists