DarkHelm.org/plex-playlist
Public Access
4
1
Fork 0
Code Issues 55 Pull Requests 3 Actions 9 Packages Projects 1 Releases Wiki Activity

Stabilize self-hosted CI workflows and resolve issue #62 #73

Merged
darkhelm merged 80 commits from feature/two-stage-runtime-images into main 2026-07-13 11:16:18 -04:00
Conversation 0 Commits 80 Files Changed 35
+24 -245
1 changed files with 24 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 383b88e87d - Show all commits
.gitea/workflows/cicd.yaml
+24 -245
View File
@@ -495,12 +495,7 @@ jobs:
if: always() && needs.build_cicd.result == 'failure'
timeout-minutes: 10
steps:
- name: Identify runner
run: |
echo "=== Runner Identity ==="
echo "runner_name=${RUNNER_NAME:-unknown}"
echo "runner_hostname=${HOSTNAME:-unknown}"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
- *identify_runner_step
- name: Build CICD postmortem diagnostics
run: |
@@ -549,7 +544,7 @@ jobs:
- &ensure_cicd_image_step
name: Ensure CICD image is available
env:
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || needs.runtime_images.outputs.head_sha || github.sha }}
HEAD_SHA: ${{ needs.build_cicd.outputs.head_sha || github.sha }}
run: |
IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
@@ -1281,226 +1276,6 @@ jobs:
- *failure_diagnostics_step
runtime_images:
name: Build and Publish Runtime Images (Legacy Disabled)
runs-on: ubuntu-act-8gb
if: false
needs: [build_cicd, cicd-tests-complete, build-backend-base-image, build-frontend-base-image]
timeout-minutes: 45
outputs:
head_sha: ${{ steps.meta.outputs.head_sha }}
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
deployable_frontend_tag_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_tag_ref }}
deployable_frontend_digest_ref: ${{ steps.deployable_frontend_ref.outputs.deployable_frontend_digest_ref }}
steps:
- *identify_runner_step
- *resolve_head_sha_from_build_step
- name: Minimal checkout for runtime image inputs
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
umask 077
trap 'rm -f ~/.ssh/id_rsa' EXIT
retry_cmd() {
attempts="${1:-5}"
backoff="${2:-2}"
shift 2
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
if "$@"; then
return 0
fi
if [ "${attempt}" -lt "${attempts}" ]; then
sleep_seconds=$((backoff * attempt))
echo "Command failed (attempt ${attempt}/${attempts}): $*"
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
echo "Command failed after ${attempts} attempts: $*"
return 1
}
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
mkdir -p ~/.ssh
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
retry_cmd 5 2 sh -c "ssh-keyscan -p '${GITEA_SSH_PORT}' '${GITEA_SSH_HOST}' >> ~/.ssh/known_hosts 2>/dev/null"
retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
if retry_cmd 5 3 env GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
git checkout FETCH_HEAD -- \
.dockerignore \
Dockerfile.backend \
Dockerfile.frontend \
backend \
frontend \
scripts/check-dockerfile-boundaries.sh \
scripts/verify-deployable-image-purity.sh
else
git checkout HEAD -- \
.dockerignore \
Dockerfile.backend \
Dockerfile.frontend \
backend \
frontend \
scripts/check-dockerfile-boundaries.sh \
scripts/verify-deployable-image-purity.sh
fi
- name: Verify deployable runtime boundaries
run: |
set -euo pipefail
bash ./scripts/check-dockerfile-boundaries.sh
- name: Build and verify deployable runtime image purity
env:
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
docker build -f Dockerfile.backend -t deployable-backend:"${HEAD_SHA}" .
docker build -f Dockerfile.frontend --target production -t deployable-frontend:"${HEAD_SHA}" .
bash ./scripts/verify-deployable-image-purity.sh --image deployable-backend:"${HEAD_SHA}" --profile backend
bash ./scripts/verify-deployable-image-purity.sh --image deployable-frontend:"${HEAD_SHA}" --profile frontend
- name: Push deployable backend runtime image
id: deployable_backend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
retry_registry_op() {
op_name="$1"
image_ref="$2"
attempts="${3:-5}"
backoff="${4:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
if [ "${op_name}" = "push" ]; then
if docker push "${image_ref}"; then
return 0
fi
else
if docker pull "${image_ref}" >/dev/null; then
return 0
fi
fi
if [ "${attempt}" -lt "${attempts}" ]; then
echo "${op_name} failed. Probing registry endpoint before retry..."
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true
sleep_seconds=$((backoff * attempt))
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
retry_registry_op push "${DEPLOYABLE_BACKEND_TAG_REF}" 5 4
retry_registry_op pull "${DEPLOYABLE_BACKEND_TAG_REF}" 5 3
DEPLOYABLE_BACKEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" | grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" | head -n 1; } || true)"
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve backend digest after push/pull retries"
exit 1
fi
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- name: Push deployable frontend runtime image
id: deployable_frontend_ref
env:
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
run: |
set -euo pipefail
retry_registry_op() {
op_name="$1"
image_ref="$2"
attempts="${3:-5}"
backoff="${4:-3}"
attempt=1
while [ "${attempt}" -le "${attempts}" ]; do
echo "${op_name} attempt ${attempt}/${attempts} for ${image_ref}"
if [ "${op_name}" = "push" ]; then
if docker push "${image_ref}"; then
return 0
fi
else
if docker pull "${image_ref}" >/dev/null; then
return 0
fi
fi
if [ "${attempt}" -lt "${attempts}" ]; then
echo "${op_name} failed. Probing registry endpoint before retry..."
timeout 10 curl -fsSIL "https://${GITEA_REGISTRY}/v2/" >/dev/null || true
sleep_seconds=$((backoff * attempt))
echo "Retrying in ${sleep_seconds}s"
sleep "${sleep_seconds}"
fi
attempt=$((attempt + 1))
done
return 1
}
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
DEPLOYABLE_FRONTEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-frontend"
DEPLOYABLE_FRONTEND_TAG_REF="${DEPLOYABLE_FRONTEND_REPO}:${HEAD_SHA}"
docker tag "deployable-frontend:${HEAD_SHA}" "${DEPLOYABLE_FRONTEND_TAG_REF}"
retry_registry_op push "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 4
retry_registry_op pull "${DEPLOYABLE_FRONTEND_TAG_REF}" 5 3
DEPLOYABLE_FRONTEND_DIGEST_REF="$({ docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_FRONTEND_TAG_REF}" | grep "^${DEPLOYABLE_FRONTEND_REPO}@sha256:" | head -n 1; } || true)"
if [ -z "${DEPLOYABLE_FRONTEND_DIGEST_REF}" ]; then
echo "❌ Unable to resolve frontend digest after push/pull retries"
exit 1
fi
echo "deployable_frontend_tag_ref=${DEPLOYABLE_FRONTEND_TAG_REF}" >> "$GITHUB_OUTPUT"
echo "deployable_frontend_digest_ref=${DEPLOYABLE_FRONTEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
- *failure_diagnostics_step
build-backend-main-image:
name: Build Backend Main Image
runs-on: ubuntu-act-8gb
@@ -1796,17 +1571,17 @@ jobs:
- *failure_diagnostics_step
runtime-images-postmortem:
name: Runtime Images Failure Postmortem
name: Production Image Failures Postmortem
# Run on the broader runner pool so we can still capture diagnostics when
# runtime_images fails during platform "Set up job" before any step executes.
# one of the active production image jobs fails during platform setup.
runs-on: ubuntu-act
needs: [runtime_images, build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
if: always() && (needs.runtime_images.result == 'failure' || needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure')
needs: [build-backend-main-image, build-frontend-main-image, build-integration-tester-image, build-e2e-tester-image]
if: always() && (needs['build-backend-main-image'].result == 'failure' || needs['build-frontend-main-image'].result == 'failure' || needs['build-integration-tester-image'].result == 'failure' || needs['build-e2e-tester-image'].result == 'failure')
timeout-minutes: 10
steps:
- *identify_runner_step
- name: Collect runtime_images postmortem context
- name: Collect production image postmortem context
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
@@ -1819,7 +1594,10 @@ jobs:
echo "run_attempt=${GITHUB_RUN_ATTEMPT:-unknown}"
echo "repository=${GITHUB_REPOSITORY:-unknown}"
echo "server_url=${GITHUB_SERVER_URL:-unknown}"
echo "job_result_runtime_images=${{ needs.runtime_images.result }}"
echo "build_backend_main_image_result=${{ needs['build-backend-main-image'].result }}"
echo "build_frontend_main_image_result=${{ needs['build-frontend-main-image'].result }}"
echo "build_integration_tester_image_result=${{ needs['build-integration-tester-image'].result }}"
echo "build_e2e_tester_image_result=${{ needs['build-e2e-tester-image'].result }}"
echo "=== Local Runner Telemetry (postmortem job host) ==="
uname -a || true
@@ -1875,20 +1653,21 @@ jobs:
' completed = _g(job, "completed_at", "end_time")' \
' print(f"job={name} status={status} conclusion={conclusion} runner={runner} started={started} completed={completed}")' \
'' \
'target = None' \
'targets = {' \
' "Build Backend Main Image",' \
' "Build Frontend Main Image",' \
' "Build Integration Tester Image",' \
' "Build E2E Tester Image",' \
'}' \
'' \
'print("=== targeted production image jobs ===")' \
'for job in jobs:' \
' name = str(_g(job, "name", "job_name"))' \
' if "Build and Publish Runtime Images" in name:' \
' target = job' \
' break' \
'' \
'if target is None:' \
' print("runtime_images_job=not_found")' \
'else:' \
' print("=== runtime_images job detail ===")' \
' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \
' if key in target:' \
' print(f"{key}={target[key]}")')"
' if name in targets:' \
' for key in ("id", "name", "status", "conclusion", "runner_name", "started_at", "completed_at", "html_url", "logs_url"):' \
' if key in job:' \
' print(f"{key}={job[key]}")' \
' print("---")')"
else
echo "python3 unavailable; emitting raw jobs payload head"
sed -n '1,120p' /tmp/actions-jobs.json || true