Stabilize self-hosted CI workflows and resolve issue #62 #73
@@ -25,6 +25,7 @@ secrets/
|
||||
.ruff_cache/
|
||||
**/__pycache__/
|
||||
**/.pytest_cache/
|
||||
**/.venv/
|
||||
**/node_modules/
|
||||
**/dist/
|
||||
**/build/
|
||||
|
||||
@@ -1,478 +0,0 @@
|
||||
name: CICD Source Checks
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
head_sha:
|
||||
description: Commit SHA to process
|
||||
required: false
|
||||
source_workflow:
|
||||
description: Upstream workflow name
|
||||
required: false
|
||||
trace_id:
|
||||
description: Correlation id propagated across CICD dispatch chain
|
||||
required: false
|
||||
base_needed:
|
||||
description: Whether base rebuild is required downstream
|
||||
required: false
|
||||
base_hash:
|
||||
description: Immutable base hash to pass downstream
|
||||
required: false
|
||||
|
||||
env:
|
||||
GITEA_SSH_HOST: kankali.darkhelm.lan
|
||||
GITEA_SSH_PORT: "2222"
|
||||
GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
|
||||
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
||||
GITEA_REGISTRY_IP: 10.18.75.2
|
||||
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
||||
|
||||
concurrency:
|
||||
group: source-checks-${{ github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
name: Setup Source Checks Context
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
head_sha: ${{ steps.meta.outputs.head_sha }}
|
||||
trace_id: ${{ steps.meta.outputs.trace_id }}
|
||||
base_needed: ${{ steps.meta.outputs.base_needed }}
|
||||
base_hash: ${{ steps.meta.outputs.base_hash }}
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Audit trigger context
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
BASE_NEEDED_INPUT: ${{ github.event.inputs.base_needed }}
|
||||
BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
RESOLVED_BASE_NEEDED="${BASE_NEEDED_INPUT:-true}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-source-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}"
|
||||
echo "=== Dispatch Audit: CICD Source Checks ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "source_workflow=${SOURCE_WORKFLOW}"
|
||||
echo "head_sha_input=${HEAD_SHA_INPUT}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}"
|
||||
echo "base_needed=${RESOLVED_BASE_NEEDED}"
|
||||
echo "base_hash=${BASE_HASH_INPUT:-deferred}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
|
||||
- name: Resolve source check metadata
|
||||
id: meta
|
||||
env:
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
BASE_NEEDED_INPUT: ${{ github.event.inputs.base_needed }}
|
||||
BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
RESOLVED_TRACE_ID="${TRACE_ID_INPUT:-cicd-source-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}"
|
||||
RESOLVED_BASE_NEEDED="${BASE_NEEDED_INPUT:-true}"
|
||||
RESOLVED_BASE_HASH="${BASE_HASH_INPUT:-deferred}"
|
||||
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "trace_id=${RESOLVED_TRACE_ID}" >> "$GITHUB_OUTPUT"
|
||||
echo "base_needed=${RESOLVED_BASE_NEEDED}" >> "$GITHUB_OUTPUT"
|
||||
echo "base_hash=${RESOLVED_BASE_HASH}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- &failure_diagnostics_step
|
||||
name: Failure diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Failure Diagnostics ==="
|
||||
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
||||
echo "runner_name=${RUNNER_NAME:-unknown}"
|
||||
echo "runner_hostname=${HOSTNAME:-unknown}"
|
||||
uname -a || true
|
||||
cat /etc/os-release 2>/dev/null || true
|
||||
df -h || true
|
||||
free -h || true
|
||||
ps aux --sort=-%mem | head -n 30 || true
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "=== Docker Diagnostics ==="
|
||||
docker version || true
|
||||
docker info || true
|
||||
docker ps -a || true
|
||||
docker images --digests | head -n 50 || true
|
||||
else
|
||||
echo "docker not available on this runner"
|
||||
fi
|
||||
|
||||
echo "=== Kernel Tail ==="
|
||||
dmesg | tail -n 120 || true
|
||||
|
||||
source-precommit-checks-backend:
|
||||
name: Source Pre-commit Checks (backend)
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 40
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Configure registry host resolution
|
||||
run: |
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
- name: Checkout source snapshot
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
set -e
|
||||
umask 022
|
||||
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
||||
|
||||
mkdir -p ~/.ssh
|
||||
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
||||
|
||||
if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
||||
git checkout FETCH_HEAD -- .
|
||||
echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}"
|
||||
else
|
||||
git checkout HEAD -- .
|
||||
echo "Falling back to default branch HEAD for source checks checkout"
|
||||
fi
|
||||
|
||||
test -f .pre-commit-config.yaml || {
|
||||
echo "❌ Missing .pre-commit-config.yaml after checkout"
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -f backend/pyproject.toml || {
|
||||
echo "❌ Missing backend/pyproject.toml after checkout"
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -f frontend/package.json || {
|
||||
echo "❌ Missing frontend/package.json after checkout"
|
||||
find . -maxdepth 3 -type f | sort | head -n 80
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Bootstrap backend toolchain
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq curl ca-certificates
|
||||
fi
|
||||
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
fi
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install backend dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
export UV_LINK_MODE=copy
|
||||
|
||||
cd backend
|
||||
uv sync --dev
|
||||
|
||||
- name: Run backend pre-commit source checks
|
||||
env:
|
||||
CI: "true"
|
||||
SKIP: "eslint,prettier,typescript-check,tsdoc-lint"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
cd backend
|
||||
uv run pre-commit run --all-files --show-diff-on-failure --config ../.pre-commit-config.yaml
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
source-precommit-checks-frontend:
|
||||
name: Source Pre-commit Checks (frontend)
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 40
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Configure registry host resolution
|
||||
run: |
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
- name: Checkout source snapshot
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
set -e
|
||||
umask 022
|
||||
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
||||
|
||||
mkdir -p ~/.ssh
|
||||
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
||||
|
||||
if GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1; then
|
||||
git checkout FETCH_HEAD -- .
|
||||
echo "Using fetched HEAD_SHA checkout: ${HEAD_SHA}"
|
||||
else
|
||||
git checkout HEAD -- .
|
||||
echo "Falling back to default branch HEAD for source checks checkout"
|
||||
fi
|
||||
|
||||
test -f .pre-commit-config.yaml || {
|
||||
echo "❌ Missing .pre-commit-config.yaml after checkout"
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -f frontend/package.json || {
|
||||
echo "❌ Missing frontend/package.json after checkout"
|
||||
find . -maxdepth 3 -type f | sort | head -n 80
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Bootstrap frontend toolchain
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq curl ca-certificates
|
||||
fi
|
||||
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
fi
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
if ! command -v node >/dev/null 2>&1; then
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq nodejs npm
|
||||
fi
|
||||
if ! command -v corepack >/dev/null 2>&1; then
|
||||
npm install -g corepack
|
||||
fi
|
||||
|
||||
corepack enable
|
||||
|
||||
- name: Install frontend dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export NODE_OPTIONS="--max-old-space-size=512"
|
||||
export YARN_NETWORK_CONCURRENCY=1
|
||||
cd frontend
|
||||
if ! yarn install --immutable --mode=skip-build; then
|
||||
echo "Yarn install failed; retrying with npm fallback for constrained runner memory"
|
||||
rm -f package-lock.json
|
||||
npm install --ignore-scripts --no-audit --no-fund --prefer-offline
|
||||
fi
|
||||
|
||||
- name: Run frontend pre-commit source checks
|
||||
env:
|
||||
CI: "true"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd frontend
|
||||
corepack yarn eslint . --max-warnings=0
|
||||
corepack yarn vue-tsc --noEmit
|
||||
corepack yarn prettier --check .
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
dispatch-build:
|
||||
name: Dispatch Downstream Build
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 10
|
||||
needs: [setup, source-precommit-checks-backend, source-precommit-checks-frontend]
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Configure registry host resolution
|
||||
run: |
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
- name: Dispatch downstream workflow
|
||||
env:
|
||||
ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }}
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
BASE_NEEDED: ${{ needs.setup.outputs.base_needed }}
|
||||
BASE_HASH: ${{ needs.setup.outputs.base_hash }}
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
TRACE_ID: ${{ needs.setup.outputs.trace_id }}
|
||||
REPO_FULL: ${{ github.repository }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -e
|
||||
|
||||
DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}"
|
||||
|
||||
if [ -z "${DISPATCH_TOKEN}" ]; then
|
||||
echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPO_OWNER="${REPO_FULL%/*}"
|
||||
REPO_NAME="${REPO_FULL#*/}"
|
||||
TARGET_REF="${HEAD_REF:-${REF_NAME}}"
|
||||
|
||||
if [ "${BASE_NEEDED}" = "true" ]; then
|
||||
TARGET_WORKFLOW="docker-build-base.yaml"
|
||||
else
|
||||
TARGET_WORKFLOW="docker-build-main.yaml"
|
||||
fi
|
||||
|
||||
echo "route_decision workflow=${TARGET_WORKFLOW} head_sha=${HEAD_SHA} base_needed=${BASE_NEEDED} trace_id=${TRACE_ID}"
|
||||
|
||||
CANDIDATE_API_BASES=()
|
||||
if [ -n "${GITHUB_SERVER_URL:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
|
||||
fi
|
||||
if [ -n "${GITEA_SSH_HOST:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1")
|
||||
fi
|
||||
if [ -n "${GITEA_REGISTRY_HOST:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
|
||||
fi
|
||||
if [ -n "${GITEA_REGISTRY_IP:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
|
||||
fi
|
||||
|
||||
ensure_curl() {
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq curl ca-certificates
|
||||
fi
|
||||
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "❌ curl is required for dispatch and could not be installed"
|
||||
return 1
|
||||
}
|
||||
|
||||
ensure_curl
|
||||
|
||||
HELPER_PATH="/tmp/dispatch-workflow.sh"
|
||||
|
||||
fetch_dispatch_helper() {
|
||||
local helper_ref="$1"
|
||||
local api_base
|
||||
for api_base in "${CANDIDATE_API_BASES[@]}"; do
|
||||
helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
|
||||
if curl -fsS --connect-timeout 5 --max-time 20 \
|
||||
-H "Authorization: token ${DISPATCH_TOKEN}" \
|
||||
-H "User-Agent: plex-playlist-cicd-source-checks" \
|
||||
-o "${HELPER_PATH}" \
|
||||
"${helper_url}"; then
|
||||
chmod +x "${HELPER_PATH}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
|
||||
echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DISPATCH_ARGS=(
|
||||
--token "${DISPATCH_TOKEN}"
|
||||
--repo "${REPO_FULL}"
|
||||
--workflow "${TARGET_WORKFLOW}"
|
||||
--ref "${TARGET_REF}"
|
||||
--head-sha "${HEAD_SHA}"
|
||||
--source-workflow "CICD Source Checks"
|
||||
--trace-id "${TRACE_ID}"
|
||||
--base-needed "${BASE_NEEDED}"
|
||||
--base-hash "${BASE_HASH}"
|
||||
)
|
||||
|
||||
for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
|
||||
DISPATCH_ARGS+=(--api-base "${API_BASE}")
|
||||
done
|
||||
|
||||
echo "✅ Source checks passed; dispatching ${TARGET_WORKFLOW}"
|
||||
"${HELPER_PATH}" "${DISPATCH_ARGS[@]}"
|
||||
|
||||
- *failure_diagnostics_step
|
||||
@@ -1,228 +0,0 @@
|
||||
name: CICD Start
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ '**' ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
trace_id:
|
||||
description: Correlation id propagated across CICD dispatch chain
|
||||
required: false
|
||||
|
||||
env:
|
||||
GITEA_SSH_HOST: kankali.darkhelm.lan
|
||||
GITEA_SSH_PORT: "2222"
|
||||
GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
|
||||
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
||||
GITEA_REGISTRY_IP: 10.18.75.2
|
||||
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
||||
|
||||
concurrency:
|
||||
group: start-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
sanity-and-routing:
|
||||
name: Sanity and Base Decision
|
||||
# Keep startup lightweight to avoid runner image setup stalls.
|
||||
# Use the same stable self-hosted label as the downstream CICD jobs.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 12
|
||||
outputs:
|
||||
base_needed: ${{ steps.base-decision.outputs.base_needed }}
|
||||
head_sha: ${{ steps.base-decision.outputs.head_sha }}
|
||||
base_hash: ${{ steps.base-decision.outputs.base_hash }}
|
||||
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Audit trigger context
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
ACTOR: ${{ github.actor }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
SHA: ${{ github.sha }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-start-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${SHA:0:8}}"
|
||||
echo "=== Dispatch Audit: CICD Start ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "repository=${REPOSITORY}"
|
||||
echo "actor=${ACTOR}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "sha=${SHA}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
|
||||
- name: Configure registry host resolution
|
||||
run: |
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
- name: Compute routing signal
|
||||
id: base-decision
|
||||
run: |
|
||||
set -e
|
||||
|
||||
# Keep this workflow container-agnostic: do not rely on checkout/actions/node.
|
||||
# docker-build-base handles canonical base hash/existence checks.
|
||||
echo "head_sha=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
|
||||
echo "base_hash=deferred" >> "${GITHUB_OUTPUT}"
|
||||
echo "base_needed=true" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
echo "Routing via docker-build-base.yaml"
|
||||
echo "Base hash delegated to Docker Build Base workflow"
|
||||
|
||||
- name: Dispatch downstream workflow
|
||||
if: steps.base-decision.outcome == 'success' && steps.base-decision.outputs.head_sha != '' && steps.base-decision.outputs.base_needed != ''
|
||||
env:
|
||||
ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }}
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
BASE_NEEDED: ${{ steps.base-decision.outputs.base_needed }}
|
||||
HEAD_SHA: ${{ steps.base-decision.outputs.head_sha }}
|
||||
BASE_HASH: ${{ steps.base-decision.outputs.base_hash }}
|
||||
REPO_FULL: ${{ github.repository }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
set -e
|
||||
|
||||
DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}"
|
||||
|
||||
if [ -z "${DISPATCH_TOKEN}" ]; then
|
||||
echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPO_OWNER="${REPO_FULL%/*}"
|
||||
REPO_NAME="${REPO_FULL#*/}"
|
||||
TARGET_REF="${HEAD_REF:-${REF_NAME}}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-start-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}"
|
||||
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
echo "target_ref=${TARGET_REF}"
|
||||
|
||||
TARGET_WORKFLOW="cicd-source-checks.yaml"
|
||||
|
||||
echo "route_decision workflow=${TARGET_WORKFLOW} head_sha=${HEAD_SHA} base_needed=${BASE_NEEDED} trace_id=${TRACE_ID}"
|
||||
|
||||
CANDIDATE_API_BASES=()
|
||||
if [ -n "${GITHUB_SERVER_URL:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
|
||||
fi
|
||||
if [ -n "${GITEA_SSH_HOST:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1")
|
||||
fi
|
||||
if [ -n "${GITEA_REGISTRY_HOST:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
|
||||
fi
|
||||
if [ -n "${GITEA_REGISTRY_IP:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
|
||||
fi
|
||||
|
||||
ensure_curl() {
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq curl ca-certificates
|
||||
fi
|
||||
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "❌ curl is required for dispatch and could not be installed"
|
||||
return 1
|
||||
}
|
||||
|
||||
ensure_curl
|
||||
|
||||
HELPER_PATH="/tmp/dispatch-workflow.sh"
|
||||
|
||||
fetch_dispatch_helper() {
|
||||
local helper_ref="$1"
|
||||
local api_base
|
||||
for api_base in "${CANDIDATE_API_BASES[@]}"; do
|
||||
helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
|
||||
if curl -fsS --connect-timeout 5 --max-time 20 \
|
||||
-H "Authorization: token ${DISPATCH_TOKEN}" \
|
||||
-H "User-Agent: plex-playlist-cicd-start" \
|
||||
-o "${HELPER_PATH}" \
|
||||
"${helper_url}"; then
|
||||
chmod +x "${HELPER_PATH}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
|
||||
echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DISPATCH_ARGS=(
|
||||
--token "${DISPATCH_TOKEN}"
|
||||
--repo "${REPO_FULL}"
|
||||
--workflow "${TARGET_WORKFLOW}"
|
||||
--ref "${TARGET_REF}"
|
||||
--head-sha "${HEAD_SHA}"
|
||||
--source-workflow "CICD Start"
|
||||
--trace-id "${TRACE_ID}"
|
||||
--base-needed "${BASE_NEEDED}"
|
||||
--base-hash "${BASE_HASH}"
|
||||
)
|
||||
|
||||
for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
|
||||
DISPATCH_ARGS+=(--api-base "${API_BASE}")
|
||||
done
|
||||
|
||||
"${HELPER_PATH}" "${DISPATCH_ARGS[@]}"
|
||||
|
||||
- name: Failure diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Failure Diagnostics ==="
|
||||
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
||||
echo "runner_name=${RUNNER_NAME:-unknown}"
|
||||
echo "runner_hostname=${HOSTNAME:-unknown}"
|
||||
uname -a || true
|
||||
cat /etc/os-release 2>/dev/null || true
|
||||
df -h || true
|
||||
free -h || true
|
||||
ps aux --sort=-%mem | head -n 30 || true
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "=== Docker Diagnostics ==="
|
||||
docker version || true
|
||||
docker info || true
|
||||
docker ps -a || true
|
||||
docker images --digests | head -n 50 || true
|
||||
else
|
||||
echo "docker not available on this runner"
|
||||
fi
|
||||
|
||||
echo "=== Kernel Tail ==="
|
||||
dmesg | tail -n 120 || true
|
||||
@@ -1,575 +0,0 @@
|
||||
name: CICD Tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
head_sha:
|
||||
description: Commit SHA to process
|
||||
required: false
|
||||
deployable_backend_tag_ref:
|
||||
description: Deployable backend image tag reference
|
||||
required: false
|
||||
deployable_backend_digest_ref:
|
||||
description: Deployable backend image digest reference
|
||||
required: false
|
||||
source_workflow:
|
||||
description: Upstream workflow name
|
||||
required: false
|
||||
trace_id:
|
||||
description: Correlation id propagated across CICD dispatch chain
|
||||
required: false
|
||||
|
||||
env:
|
||||
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
||||
GITEA_REGISTRY_IP: 10.18.75.2
|
||||
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
||||
|
||||
concurrency:
|
||||
group: tests-${{ github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
name: Setup Tests Context
|
||||
# Use the same stable runner label as the test jobs.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 8
|
||||
outputs:
|
||||
head_sha: ${{ steps.meta.outputs.head_sha }}
|
||||
deployable_backend_tag_ref: ${{ steps.meta.outputs.deployable_backend_tag_ref }}
|
||||
deployable_backend_digest_ref: ${{ steps.meta.outputs.deployable_backend_digest_ref }}
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Audit trigger context
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }}
|
||||
DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}"
|
||||
echo "=== Dispatch Audit: CICD Tests ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "source_workflow=${SOURCE_WORKFLOW}"
|
||||
echo "head_sha_input=${HEAD_SHA_INPUT}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}"
|
||||
echo "deployable_backend_tag_ref_input=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}"
|
||||
echo "deployable_backend_digest_ref_input=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
|
||||
- name: Resolve head SHA
|
||||
id: meta
|
||||
env:
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
DEPLOYABLE_BACKEND_TAG_REF_INPUT: ${{ github.event.inputs.deployable_backend_tag_ref }}
|
||||
DEPLOYABLE_BACKEND_DIGEST_REF_INPUT: ${{ github.event.inputs.deployable_backend_digest_ref }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF_INPUT}" >> "$GITHUB_OUTPUT"
|
||||
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF_INPUT}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- &failure_diagnostics_step
|
||||
name: Failure diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Failure Diagnostics ==="
|
||||
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
||||
echo "runner_name=${RUNNER_NAME:-unknown}"
|
||||
echo "runner_hostname=${HOSTNAME:-unknown}"
|
||||
uname -a || true
|
||||
cat /etc/os-release 2>/dev/null || true
|
||||
df -h || true
|
||||
free -h || true
|
||||
ps aux --sort=-%mem | head -n 30 || true
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "=== Docker Diagnostics ==="
|
||||
docker version || true
|
||||
docker info || true
|
||||
docker ps -a || true
|
||||
docker images --digests | head -n 50 || true
|
||||
else
|
||||
echo "docker not available on this runner"
|
||||
fi
|
||||
|
||||
echo "=== Kernel Tail ==="
|
||||
dmesg | tail -n 120 || true
|
||||
|
||||
backend-tests:
|
||||
name: Backend Tests
|
||||
# Use ubuntu-act runner pool for consistent availability.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 25
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- &configure_registry_host_step
|
||||
name: Configure registry host resolution
|
||||
run: |
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
- &ensure_cicd_image_step
|
||||
name: Ensure CICD image is available
|
||||
env:
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
||||
if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
|
||||
echo "Using cached CICD image: ${IMAGE}"
|
||||
else
|
||||
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
|
||||
pulled=false
|
||||
for i in 1 2 3; do
|
||||
echo "Pull attempt ${i}/3 for ${IMAGE}"
|
||||
if docker pull "${IMAGE}"; then
|
||||
pulled=true
|
||||
break
|
||||
fi
|
||||
if [ "${i}" -lt 3 ]; then
|
||||
sleep_seconds=$((5 * i))
|
||||
echo "Pull failed; retrying in ${sleep_seconds}s"
|
||||
sleep "${sleep_seconds}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${pulled}" != "true" ]; then
|
||||
echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Run backend tests with coverage
|
||||
env:
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
set -o pipefail
|
||||
LOG_FILE="$(mktemp)"
|
||||
|
||||
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
||||
cd /workspace/backend &&
|
||||
source .venv/bin/activate &&
|
||||
uv run pytest -v --tb=short --cov=src --cov-report=term-missing --cov-fail-under=95
|
||||
" 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
TEST_STATUS=${PIPESTATUS[0]}
|
||||
if [ "${TEST_STATUS}" -ne 0 ]; then
|
||||
echo "❌ Backend tests failed (exit=${TEST_STATUS})"
|
||||
echo "--- Last 200 lines of backend test output ---"
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
exit "${TEST_STATUS}"
|
||||
fi
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
frontend-tests:
|
||||
name: Frontend Tests
|
||||
# Use ubuntu-act runner pool for consistent availability.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 25
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- *configure_registry_host_step
|
||||
- *ensure_cicd_image_step
|
||||
- name: Run frontend tests with coverage
|
||||
env:
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
set -o pipefail
|
||||
LOG_FILE="$(mktemp)"
|
||||
|
||||
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
||||
cd /workspace/frontend &&
|
||||
yarn test:coverage --run --reporter=verbose --coverage.reporter=text --coverage.reporter=text-summary --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85
|
||||
" 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
TEST_STATUS=${PIPESTATUS[0]}
|
||||
if [ "${TEST_STATUS}" -ne 0 ]; then
|
||||
echo "❌ Frontend tests failed (exit=${TEST_STATUS})"
|
||||
echo "--- Last 200 lines of frontend test output ---"
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
exit "${TEST_STATUS}"
|
||||
fi
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
xdoctest:
|
||||
name: Backend Doctests
|
||||
# Use ubuntu-act runner pool for consistent availability.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 15
|
||||
needs: setup
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- *configure_registry_host_step
|
||||
- *ensure_cicd_image_step
|
||||
- name: Run backend doctests
|
||||
env:
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
set -o pipefail
|
||||
LOG_FILE="$(mktemp)"
|
||||
|
||||
docker run --rm "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
||||
cd /workspace/backend &&
|
||||
source .venv/bin/activate &&
|
||||
uv run xdoctest src/ --quiet
|
||||
" 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
TEST_STATUS=${PIPESTATUS[0]}
|
||||
if [ "${TEST_STATUS}" -ne 0 ]; then
|
||||
echo "❌ Backend doctests failed (exit=${TEST_STATUS})"
|
||||
echo "--- Last 200 lines of doctest output ---"
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
exit "${TEST_STATUS}"
|
||||
fi
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
integration-tests:
|
||||
name: Runtime Black-Box Integration Tests
|
||||
# Pin integration tests to high-memory worker to reduce setup-stage runner churn.
|
||||
runs-on: ubuntu-act-8gb
|
||||
timeout-minutes: 20
|
||||
needs: [setup, backend-tests]
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- *configure_registry_host_step
|
||||
- name: Run runtime black-box integration checks
|
||||
env:
|
||||
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.setup.outputs.deployable_backend_tag_ref }}
|
||||
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.setup.outputs.deployable_backend_digest_ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set -o pipefail
|
||||
|
||||
RUN_ID="${GITHUB_RUN_ID:-local}"
|
||||
RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT:-1}"
|
||||
LOG_FILE="$(mktemp)"
|
||||
NETWORK_NAME="plex-blackbox-${RUN_ID}-${RUN_ATTEMPT}"
|
||||
DB_CONTAINER="plex-blackbox-db-${RUN_ID}-${RUN_ATTEMPT}"
|
||||
BACKEND_CONTAINER="plex-blackbox-backend-${RUN_ID}-${RUN_ATTEMPT}"
|
||||
DB_PASSWORD="plex_password"
|
||||
DB_URL="postgresql://plex_user:${DB_PASSWORD}@${DB_CONTAINER}:5432/plex_playlist"
|
||||
|
||||
fetch_backend_endpoint() {
|
||||
endpoint_path="$1"
|
||||
output_file="$2"
|
||||
probe_tmp="$(mktemp)"
|
||||
|
||||
if ! docker exec "${BACKEND_CONTAINER}" python -c "$(printf '%s\n' \
|
||||
'import sys' \
|
||||
'import urllib.error' \
|
||||
'import urllib.request' \
|
||||
'endpoint_path = sys.argv[1]' \
|
||||
'url = f"http://127.0.0.1:8000{endpoint_path}"' \
|
||||
'try:' \
|
||||
' with urllib.request.urlopen(url, timeout=2) as response:' \
|
||||
' body = response.read().decode("utf-8", errors="replace")' \
|
||||
' print(response.status)' \
|
||||
' print(body)' \
|
||||
'except urllib.error.HTTPError as err:' \
|
||||
' body = err.read().decode("utf-8", errors="replace")' \
|
||||
' print(err.code)' \
|
||||
' print(body)' \
|
||||
'except Exception:' \
|
||||
' print("000")' \
|
||||
' print("")'
|
||||
)" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null
|
||||
then
|
||||
: >"${output_file}"
|
||||
rm -f "${probe_tmp}"
|
||||
echo "000"
|
||||
return 0
|
||||
fi
|
||||
|
||||
http_code="$(head -n 1 "${probe_tmp}")"
|
||||
tail -n +2 "${probe_tmp}" >"${output_file}"
|
||||
rm -f "${probe_tmp}"
|
||||
echo "${http_code}"
|
||||
}
|
||||
|
||||
dump_failure_context() {
|
||||
echo "=== Runtime Integration Failure Context ==="
|
||||
docker ps -a || true
|
||||
echo "--- Backend logs (tail 200) ---"
|
||||
docker logs "${BACKEND_CONTAINER}" 2>&1 | tail -n 200 || true
|
||||
echo "--- Database logs (tail 200) ---"
|
||||
docker logs "${DB_CONTAINER}" 2>&1 | tail -n 200 || true
|
||||
echo "--- Backend inspect (status/image) ---"
|
||||
docker inspect --format '{{json .State}} {{.Image}}' "${BACKEND_CONTAINER}" || true
|
||||
echo "--- Database inspect (status/image) ---"
|
||||
docker inspect --format '{{json .State}} {{.Image}}' "${DB_CONTAINER}" || true
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "${BACKEND_CONTAINER}" >/dev/null 2>&1 || true
|
||||
docker rm -f "${DB_CONTAINER}" >/dev/null 2>&1 || true
|
||||
docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
{
|
||||
if [ -z "${DEPLOYABLE_BACKEND_TAG_REF}" ] || [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
||||
echo "❌ Missing deployable backend image references from dispatch inputs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
docker pull "${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
docker pull "${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
||||
|
||||
DEPLOYABLE_BACKEND_REPO="${DEPLOYABLE_BACKEND_DIGEST_REF%%@*}"
|
||||
EXPECTED_DIGEST_REF="${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
||||
ACTUAL_TAG_DIGEST_REF="$({
|
||||
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
|
||||
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
|
||||
| head -n 1
|
||||
} || true)"
|
||||
|
||||
if [ -z "${ACTUAL_TAG_DIGEST_REF}" ]; then
|
||||
echo "❌ Could not resolve digest from tag reference: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${ACTUAL_TAG_DIGEST_REF}" != "${EXPECTED_DIGEST_REF}" ]; then
|
||||
echo "❌ Tag and digest mismatch"
|
||||
echo "expected=${EXPECTED_DIGEST_REF}"
|
||||
echo "actual=${ACTUAL_TAG_DIGEST_REF}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Resolved deployable backend tag: ${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
echo "Resolved deployable backend digest: ${EXPECTED_DIGEST_REF}"
|
||||
|
||||
docker network create "${NETWORK_NAME}"
|
||||
|
||||
docker run -d \
|
||||
--name "${DB_CONTAINER}" \
|
||||
--network "${NETWORK_NAME}" \
|
||||
-e POSTGRES_DB=plex_playlist \
|
||||
-e POSTGRES_USER=plex_user \
|
||||
-e POSTGRES_PASSWORD="${DB_PASSWORD}" \
|
||||
postgres:16-alpine
|
||||
|
||||
db_ready=false
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "${DB_CONTAINER}" pg_isready -U plex_user -d plex_playlist >/dev/null 2>&1; then
|
||||
db_ready=true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [ "${db_ready}" != "true" ]; then
|
||||
echo "❌ Database did not become ready"
|
||||
dump_failure_context
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker run -d \
|
||||
--name "${BACKEND_CONTAINER}" \
|
||||
--network "${NETWORK_NAME}" \
|
||||
-p 18000:8000 \
|
||||
-e DATABASE_URL="${DB_URL}" \
|
||||
-e ENVIRONMENT=production \
|
||||
"${EXPECTED_DIGEST_REF}"
|
||||
|
||||
backend_ready=false
|
||||
for i in $(seq 1 40); do
|
||||
backend_running="$(docker inspect -f '{{.State.Running}}' "${BACKEND_CONTAINER}" 2>/dev/null || echo false)"
|
||||
if [ "${backend_running}" != "true" ]; then
|
||||
echo "❌ Backend container exited before becoming healthy"
|
||||
dump_failure_context
|
||||
exit 1
|
||||
fi
|
||||
|
||||
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
||||
if [ "${health_code}" = "200" ]; then
|
||||
backend_ready=true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [ "${backend_ready}" != "true" ]; then
|
||||
echo "❌ Backend did not become healthy"
|
||||
cat /tmp/blackbox-health.json 2>/dev/null || true
|
||||
dump_failure_context
|
||||
exit 1
|
||||
fi
|
||||
|
||||
root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)"
|
||||
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
|
||||
echo "❌ Root endpoint validation failed"
|
||||
cat /tmp/blackbox-root.json 2>/dev/null || true
|
||||
dump_failure_context
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)"
|
||||
if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
|
||||
echo "❌ Compatibility endpoint validation failed"
|
||||
cat /tmp/blackbox-compatibility.json 2>/dev/null || true
|
||||
dump_failure_context
|
||||
exit 1
|
||||
fi
|
||||
|
||||
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
||||
if [ "${health_code}" != "200" ] || ! grep -q '"status":"healthy"' /tmp/blackbox-health.json; then
|
||||
echo "❌ Health endpoint validation failed"
|
||||
cat /tmp/blackbox-health.json 2>/dev/null || true
|
||||
dump_failure_context
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Runtime black-box integration checks passed"
|
||||
} 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
TEST_STATUS=${PIPESTATUS[0]}
|
||||
if [ "${TEST_STATUS}" -ne 0 ]; then
|
||||
echo "❌ Runtime black-box integration checks failed (exit=${TEST_STATUS})"
|
||||
echo "--- Last 200 lines of runtime black-box integration output ---"
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
exit "${TEST_STATUS}"
|
||||
fi
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
e2e-tests:
|
||||
name: End-to-End Tests
|
||||
# Use ubuntu-act runner pool for consistent availability.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 30
|
||||
needs: [setup, frontend-tests]
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- *configure_registry_host_step
|
||||
- *ensure_cicd_image_step
|
||||
- name: Run E2E tests
|
||||
env:
|
||||
HEAD_SHA: ${{ needs.setup.outputs.head_sha }}
|
||||
run: |
|
||||
set -o pipefail
|
||||
LOG_FILE="$(mktemp)"
|
||||
|
||||
docker run --rm -e CI=true "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}" bash -c "
|
||||
cd /workspace/frontend &&
|
||||
if [ -d 'tests/e2e' ] || grep -q 'playwright' package.json; then
|
||||
yarn playwright --version &&
|
||||
PW_BROWSER_PATH="\${PLAYWRIGHT_BROWSERS_PATH:-/root/.cache/ms-playwright}" &&
|
||||
if find "\${PW_BROWSER_PATH}" -maxdepth 1 -type d -name 'chromium-*' | grep -q .; then
|
||||
echo 'Using preinstalled Playwright Chromium from '"\${PW_BROWSER_PATH}"
|
||||
else
|
||||
browser_ok=false &&
|
||||
for i in 1 2 3; do
|
||||
echo 'Playwright browser install attempt' "\$i"'/3' &&
|
||||
if timeout 1800 yarn playwright install chromium; then
|
||||
browser_ok=true
|
||||
break
|
||||
fi
|
||||
if [ "\$i" -lt 3 ]; then
|
||||
echo 'Playwright install attempt failed; retrying in 20s'
|
||||
sleep 20
|
||||
fi
|
||||
done &&
|
||||
if [ "\$browser_ok" != 'true' ]; then
|
||||
echo '❌ Playwright browser install failed after 3 attempts'
|
||||
exit 1
|
||||
fi
|
||||
fi &&
|
||||
yarn test:e2e --reporter=list --timeout=90000
|
||||
else
|
||||
echo 'No E2E tests found'
|
||||
fi
|
||||
" 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
TEST_STATUS=${PIPESTATUS[0]}
|
||||
if [ "${TEST_STATUS}" -ne 0 ]; then
|
||||
echo "❌ E2E tests failed (exit=${TEST_STATUS})"
|
||||
echo "--- Last 200 lines of E2E output ---"
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
exit "${TEST_STATUS}"
|
||||
fi
|
||||
|
||||
- *failure_diagnostics_step
|
||||
2432
.gitea/workflows/cicd.yaml
Normal file
2432
.gitea/workflows/cicd.yaml
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,537 +0,0 @@
|
||||
name: Docker Build Base
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_rebuild:
|
||||
description: Force a rebuild even when the immutable base tag already exists
|
||||
required: false
|
||||
default: "false"
|
||||
head_sha:
|
||||
description: Commit SHA to process
|
||||
required: false
|
||||
source_workflow:
|
||||
description: Upstream workflow name
|
||||
required: false
|
||||
trace_id:
|
||||
description: Correlation id propagated across CICD dispatch chain
|
||||
required: false
|
||||
|
||||
concurrency:
|
||||
group: base-${{ github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITEA_SSH_HOST: kankali.darkhelm.lan
|
||||
GITEA_SSH_PORT: "2222"
|
||||
GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
|
||||
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
||||
GITEA_REGISTRY_IP: 10.18.75.2
|
||||
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
publish-base:
|
||||
name: Build and Publish CICD Base Image
|
||||
# Use ubuntu-act runner pool which is known to be available.
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 35
|
||||
outputs:
|
||||
base_hash: ${{ steps.base-state.outputs.base_hash }}
|
||||
head_sha: ${{ steps.meta.outputs.head_sha }}
|
||||
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Emit startup diagnostics
|
||||
shell: sh
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
|
||||
FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
TARGET_LABEL: ubuntu-act
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
TRACE_SUFFIX="$(printf '%s' "${RESOLVED_HEAD_SHA}" | cut -c1-8)"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TRACE_SUFFIX}}"
|
||||
echo "=== Base Workflow Startup Audit ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "source_workflow=${SOURCE_WORKFLOW}"
|
||||
echo "force_rebuild=${FORCE_REBUILD}"
|
||||
echo "head_sha_input=${HEAD_SHA_INPUT}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
echo "target_runner_label=${TARGET_LABEL}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "startup_audit=ok"
|
||||
|
||||
- name: Preflight Docker and registry connectivity
|
||||
env:
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
run: |
|
||||
set -e
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
echo "=== Docker Preflight ==="
|
||||
command -v docker
|
||||
docker --version
|
||||
docker info >/tmp/docker-info.log 2>&1 || (echo "❌ docker info failed" && tail -n 40 /tmp/docker-info.log && exit 1)
|
||||
grep -qi "${GITEA_REGISTRY}" /tmp/docker-info.log || echo "⚠ Registry not listed in docker info insecure registries: ${GITEA_REGISTRY}"
|
||||
|
||||
echo "=== Registry Login Preflight ==="
|
||||
if [ -z "${PACKAGE_ACCESS_TOKEN}" ]; then
|
||||
echo "❌ PACKAGE_ACCESS_TOKEN is empty"
|
||||
exit 1
|
||||
fi
|
||||
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/tmp/docker-login.log 2>&1 || (echo "❌ Registry login preflight failed" && tail -n 40 /tmp/docker-login.log && exit 1)
|
||||
echo "registry_login_preflight=ok"
|
||||
|
||||
- name: Audit trigger context
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
|
||||
FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}"
|
||||
echo "=== Dispatch Audit: CICD Base Image ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "source_workflow=${SOURCE_WORKFLOW}"
|
||||
echo "force_rebuild=${FORCE_REBUILD}"
|
||||
echo "head_sha_input=${HEAD_SHA_INPUT}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
|
||||
- name: Resolve head SHA
|
||||
id: meta
|
||||
env:
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Minimal checkout for base inputs
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
GITHUB_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
run: |
|
||||
umask 077
|
||||
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
||||
echo "=== Minimal Repository Checkout for Base Inputs ==="
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
if [ -n "${SSH_PRIVATE_KEY}" ]; then
|
||||
mkdir -p ~/.ssh
|
||||
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
echo "Loaded SSH key fingerprint: $(ssh-keygen -lf ~/.ssh/id_rsa | awk '{print $2}')"
|
||||
ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
else
|
||||
echo "❌ SSH_PRIVATE_KEY is empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git clone --depth 1 --no-checkout \
|
||||
"${GITEA_REPO_SSH_URL}" .
|
||||
|
||||
if [ -n "${GITHUB_SHA}" ] && \
|
||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git fetch --depth 1 origin "${GITHUB_SHA}" >/dev/null 2>&1; then
|
||||
git checkout FETCH_HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh
|
||||
echo "✓ Checked out base inputs from commit ${GITHUB_SHA}"
|
||||
else
|
||||
git checkout HEAD -- Dockerfile.cicd-base .dockerignore scripts/compute-cicd-base-hash.sh
|
||||
echo "⚠ Falling back to default branch HEAD for base inputs checkout"
|
||||
fi
|
||||
|
||||
chmod +x scripts/compute-cicd-base-hash.sh
|
||||
rm -f ~/.ssh/id_rsa
|
||||
|
||||
- name: Compute base hash and inspect registry
|
||||
id: base-state
|
||||
env:
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
|
||||
run: |
|
||||
echo "=== Computing CICD Base Image Hash ==="
|
||||
|
||||
BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
|
||||
BASE_REF_HASH="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
|
||||
BASE_REF_LATEST="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:latest"
|
||||
|
||||
echo "Base hash: ${BASE_HASH}"
|
||||
echo "base_hash=${BASE_HASH}" >> $GITHUB_OUTPUT
|
||||
echo "base_ref_hash=${BASE_REF_HASH}" >> $GITHUB_OUTPUT
|
||||
echo "base_ref_latest=${BASE_REF_LATEST}" >> $GITHUB_OUTPUT
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin >/dev/null 2>&1; then
|
||||
echo "❌ Registry login failed during base inspection"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${FORCE_REBUILD}" = "true" ]; then
|
||||
echo "Manual force rebuild requested"
|
||||
echo "needs_build=true" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for i in 1 2 3; do
|
||||
echo "Existing base check ${i}/3 for ${BASE_REF_HASH}..."
|
||||
# Manifest inspect is unreliable on some runners; use pull as truth.
|
||||
if timeout 900 docker pull "${BASE_REF_HASH}" >/tmp/base-state-pull.log 2>&1; then
|
||||
echo "✓ Immutable base image already exists and is pullable: ${BASE_REF_HASH}"
|
||||
echo "needs_build=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
else
|
||||
pull_exit=$?
|
||||
echo "Existing base pull check failed with exit code ${pull_exit}"
|
||||
if [ "${pull_exit}" -eq 124 ]; then
|
||||
echo "Existing base pull timed out after 900s"
|
||||
fi
|
||||
if [ -s /tmp/base-state-pull.log ]; then
|
||||
tail -n 20 /tmp/base-state-pull.log || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${i}" -lt 3 ]; then
|
||||
sleep 10
|
||||
fi
|
||||
done
|
||||
|
||||
echo "ℹ Immutable base image missing or not yet pullable: ${BASE_REF_HASH}"
|
||||
echo "needs_build=true" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Write registry push helpers
|
||||
run: |
|
||||
cat > /tmp/registry-push-helpers.sh <<'EOF'
|
||||
#!/bin/bash
|
||||
|
||||
ensure_skopeo() {
|
||||
if command -v skopeo >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update && apt-get install -y skopeo
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache skopeo
|
||||
fi
|
||||
|
||||
if ! command -v skopeo >/dev/null 2>&1; then
|
||||
echo "❌ skopeo not available for fallback push"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
push_ref_with_fallback() {
|
||||
ref="$1"
|
||||
local_image="$2"
|
||||
archive_path="$3"
|
||||
registry_user="$4"
|
||||
package_access_token="$5"
|
||||
|
||||
for i in 1 2 3; do
|
||||
echo "Docker push attempt ${i}/3 for ${ref}..."
|
||||
if docker push "${ref}"; then
|
||||
echo "✓ Docker push succeeded for ${ref}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "${i}" -lt 3 ]; then
|
||||
sleep 10
|
||||
fi
|
||||
done
|
||||
|
||||
echo "⚠ Docker push failed for ${ref}; trying skopeo fallback"
|
||||
|
||||
if ! ensure_skopeo; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${archive_path}" ]; then
|
||||
echo "Creating local image archive for fallback push..."
|
||||
if ! docker save "${local_image}" -o "${archive_path}"; then
|
||||
echo "❌ Failed to create docker archive for fallback push"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if skopeo copy \
|
||||
--dest-creds "${registry_user}:${package_access_token}" \
|
||||
--dest-tls-verify=false \
|
||||
"docker-archive:${archive_path}" \
|
||||
"docker://${ref}"; then
|
||||
echo "✓ Skopeo fallback push succeeded for ${ref}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "❌ Skopeo fallback push failed for ${ref}"
|
||||
return 1
|
||||
}
|
||||
EOF
|
||||
|
||||
chmod 700 /tmp/registry-push-helpers.sh
|
||||
|
||||
- name: Build and push base image
|
||||
if: steps.base-state.outputs.needs_build == 'true'
|
||||
env:
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
BASE_HASH: ${{ steps.base-state.outputs.base_hash }}
|
||||
BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }}
|
||||
BASE_REF_LATEST: ${{ steps.base-state.outputs.base_ref_latest }}
|
||||
run: |
|
||||
echo "=== Building CICD Base Image ==="
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
||||
echo "❌ Registry login failed before build"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLAYWRIGHT_BROWSERS_MIRROR_TAG="${GITEA_REGISTRY}/darkhelm.org/playwright-browsers:v1.56.1-jammy"
|
||||
echo "playwright_browsers_mirror_tag=${PLAYWRIGHT_BROWSERS_MIRROR_TAG}"
|
||||
|
||||
if ! docker pull "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" >/tmp/playwright-mirror-pull.log 2>&1; then
|
||||
echo "❌ Required internal Playwright browsers mirror image is missing: ${PLAYWRIGHT_BROWSERS_MIRROR_TAG}"
|
||||
echo "Mirror this image into your internal registry before running base builds."
|
||||
tail -n 80 /tmp/playwright-mirror-pull.log || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PLAYWRIGHT_BROWSERS_IMAGE=$(docker image inspect --format='{{index .RepoDigests 0}}' "${PLAYWRIGHT_BROWSERS_MIRROR_TAG}" 2>/dev/null || true)
|
||||
if [ -z "${PLAYWRIGHT_BROWSERS_IMAGE}" ]; then
|
||||
echo "❌ Failed to resolve immutable digest for ${PLAYWRIGHT_BROWSERS_MIRROR_TAG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "playwright_browsers_image=${PLAYWRIGHT_BROWSERS_IMAGE}"
|
||||
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
BUILD_TIMEOUT_SECONDS=5400
|
||||
BUILD_START_EPOCH=$(date +%s)
|
||||
echo "docker_build_timeout_seconds=${BUILD_TIMEOUT_SECONDS}"
|
||||
echo "docker_build_started_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
timeout "${BUILD_TIMEOUT_SECONDS}" docker build --progress=plain -f Dockerfile.cicd-base \
|
||||
--build-arg BASE_IMAGE_VERSION="v1.0.0-${BASE_HASH}" \
|
||||
--build-arg BASE_IMAGE_HASH="${BASE_HASH}" \
|
||||
--build-arg PLAYWRIGHT_BROWSERS_IMAGE="${PLAYWRIGHT_BROWSERS_IMAGE}" \
|
||||
-t cicd-base:latest .
|
||||
BUILD_EXIT_CODE=$?
|
||||
if [ "${BUILD_EXIT_CODE}" -ne 0 ]; then
|
||||
if [ "${BUILD_EXIT_CODE}" -eq 124 ]; then
|
||||
echo "❌ docker build timed out after ${BUILD_TIMEOUT_SECONDS}s"
|
||||
else
|
||||
echo "❌ docker build failed with exit code ${BUILD_EXIT_CODE}"
|
||||
fi
|
||||
exit "${BUILD_EXIT_CODE}"
|
||||
fi
|
||||
|
||||
BUILD_END_EPOCH=$(date +%s)
|
||||
BUILD_ELAPSED=$((BUILD_END_EPOCH - BUILD_START_EPOCH))
|
||||
echo "docker_build_elapsed_seconds=${BUILD_ELAPSED}"
|
||||
|
||||
ARCHIVE_PATH="/tmp/cicd-base.tar"
|
||||
source /tmp/registry-push-helpers.sh
|
||||
|
||||
docker tag cicd-base:latest "${BASE_REF_HASH}"
|
||||
docker tag cicd-base:latest "${BASE_REF_LATEST}"
|
||||
|
||||
push_ref_with_fallback "${BASE_REF_HASH}" "cicd-base:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}"
|
||||
push_ref_with_fallback "${BASE_REF_LATEST}" "cicd-base:latest" "${ARCHIVE_PATH}" "${REGISTRY_USER}" "${PACKAGE_ACCESS_TOKEN}"
|
||||
|
||||
- name: Verify published base image
|
||||
env:
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
BASE_REF_HASH: ${{ steps.base-state.outputs.base_ref_hash }}
|
||||
run: |
|
||||
echo "=== Verifying Published CICD Base Image ==="
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
if ! echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin; then
|
||||
echo "❌ Registry login failed during verification"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for i in 1 2 3 4 5 6; do
|
||||
echo "Verification attempt ${i}/6 for ${BASE_REF_HASH}..."
|
||||
|
||||
if docker pull "${BASE_REF_HASH}"; then
|
||||
echo "✓ Published base image is pullable: ${BASE_REF_HASH}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${i}" -lt 6 ]; then
|
||||
echo "⚠ Pull failed; waiting 20s before retry"
|
||||
sleep 20
|
||||
fi
|
||||
done
|
||||
|
||||
echo "❌ Published base image could not be pulled after 6 attempts: ${BASE_REF_HASH}"
|
||||
exit 1
|
||||
|
||||
- name: Dispatch main build workflow
|
||||
env:
|
||||
ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }}
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
BASE_HASH: ${{ steps.base-state.outputs.base_hash }}
|
||||
REPO_FULL: ${{ github.repository }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
set -e
|
||||
|
||||
DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}"
|
||||
|
||||
if [ -z "${DISPATCH_TOKEN}" ]; then
|
||||
echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
REPO_OWNER="${REPO_FULL%/*}"
|
||||
REPO_NAME="${REPO_FULL#*/}"
|
||||
TARGET_REF="${HEAD_REF:-${REF_NAME}}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-base-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}"
|
||||
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
echo "target_ref=${TARGET_REF}"
|
||||
|
||||
CANDIDATE_API_BASES=()
|
||||
if [ -n "${GITHUB_SERVER_URL:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
|
||||
fi
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq curl ca-certificates
|
||||
fi
|
||||
|
||||
HELPER_PATH="/tmp/dispatch-workflow.sh"
|
||||
|
||||
fetch_dispatch_helper() {
|
||||
local helper_ref="$1"
|
||||
local api_base
|
||||
for api_base in "${CANDIDATE_API_BASES[@]}"; do
|
||||
helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
|
||||
if curl -fsS --connect-timeout 5 --max-time 20 \
|
||||
-H "Authorization: token ${DISPATCH_TOKEN}" \
|
||||
-H "User-Agent: plex-playlist-cicd-base" \
|
||||
-o "${HELPER_PATH}" \
|
||||
"${helper_url}"; then
|
||||
chmod +x "${HELPER_PATH}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
|
||||
echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DISPATCH_ARGS=(
|
||||
--token "${DISPATCH_TOKEN}"
|
||||
--repo "${REPO_FULL}"
|
||||
--workflow "docker-build-main.yaml"
|
||||
--ref "${TARGET_REF}"
|
||||
--head-sha "${HEAD_SHA}"
|
||||
--source-workflow "CICD Base Image"
|
||||
--trace-id "${TRACE_ID}"
|
||||
--base-needed "true"
|
||||
--base-hash "${BASE_HASH}"
|
||||
)
|
||||
|
||||
for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
|
||||
DISPATCH_ARGS+=(--api-base "${API_BASE}")
|
||||
done
|
||||
|
||||
"${HELPER_PATH}" "${DISPATCH_ARGS[@]}"
|
||||
|
||||
- name: Failure diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Failure Diagnostics ==="
|
||||
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
||||
echo "runner_name=${RUNNER_NAME:-unknown}"
|
||||
echo "runner_hostname=${HOSTNAME:-unknown}"
|
||||
uname -a || true
|
||||
cat /etc/os-release 2>/dev/null || true
|
||||
df -h || true
|
||||
free -h || true
|
||||
ps aux --sort=-%mem | head -n 30 || true
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "=== Docker Diagnostics ==="
|
||||
docker version || true
|
||||
docker info || true
|
||||
docker ps -a || true
|
||||
docker images --digests | head -n 50 || true
|
||||
else
|
||||
echo "docker not available on this runner"
|
||||
fi
|
||||
|
||||
echo "=== Kernel Tail ==="
|
||||
dmesg | tail -n 120 || true
|
||||
@@ -1,481 +0,0 @@
|
||||
name: Docker Build Main
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
head_sha:
|
||||
description: Commit SHA to process
|
||||
required: false
|
||||
base_hash:
|
||||
description: Immutable base hash to use for CICD base image
|
||||
required: false
|
||||
source_workflow:
|
||||
description: Upstream workflow name
|
||||
required: false
|
||||
base_needed:
|
||||
description: Whether base rebuild was required
|
||||
required: false
|
||||
trace_id:
|
||||
description: Correlation id propagated across CICD dispatch chain
|
||||
required: false
|
||||
|
||||
env:
|
||||
GITEA_SSH_HOST: kankali.darkhelm.lan
|
||||
GITEA_SSH_PORT: "2222"
|
||||
GITEA_REPO_SSH_URL: ssh://git@kankali.darkhelm.lan:2222/DarkHelm.org/plex-playlist.git
|
||||
GITEA_REGISTRY: kankali.darkhelm.lan:3001
|
||||
GITEA_REGISTRY_IP: 10.18.75.2
|
||||
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
concurrency:
|
||||
group: main-build-${{ github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
startup-audit:
|
||||
name: Main Workflow Startup Audit
|
||||
# Pin startup audit to high-memory worker to avoid setup-stage runner churn.
|
||||
runs-on: ubuntu-act-8gb
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Identify runner
|
||||
shell: sh
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Emit startup diagnostics
|
||||
shell: sh
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
|
||||
BASE_NEEDED: ${{ github.event.inputs.base_needed }}
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
TARGET_LABEL: ubuntu-act-8gb
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
TRACE_SUFFIX="$(printf '%s' "${RESOLVED_HEAD_SHA}" | cut -c1-8)"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TRACE_SUFFIX}}"
|
||||
echo "=== Main Workflow Startup Audit ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "source_workflow=${SOURCE_WORKFLOW}"
|
||||
echo "base_needed=${BASE_NEEDED}"
|
||||
echo "head_sha_input=${HEAD_SHA_INPUT}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}"
|
||||
echo "base_hash_input=${BASE_HASH_INPUT}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
echo "target_runner_label=${TARGET_LABEL}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "startup_audit=ok"
|
||||
|
||||
- name: Check control-plane reachability
|
||||
shell: sh
|
||||
run: |
|
||||
SERVER_URL="${GITHUB_SERVER_URL:-}"
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
if [ -n "${SERVER_URL}" ] && curl -fsS --connect-timeout 5 --max-time 10 "${SERVER_URL%/}/api/v1/version" >/tmp/gitea-version.json 2>/dev/null; then
|
||||
echo "gitea_api_reachable=true"
|
||||
cat /tmp/gitea-version.json || true
|
||||
else
|
||||
echo "gitea_api_reachable=false"
|
||||
fi
|
||||
else
|
||||
echo "curl unavailable; skipping reachability check"
|
||||
fi
|
||||
|
||||
- &failure_diagnostics_step
|
||||
name: Failure diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Failure Diagnostics ==="
|
||||
date -u '+timestamp_utc=%Y-%m-%dT%H:%M:%SZ'
|
||||
echo "runner_name=${RUNNER_NAME:-unknown}"
|
||||
echo "runner_hostname=${HOSTNAME:-unknown}"
|
||||
uname -a || true
|
||||
cat /etc/os-release 2>/dev/null || true
|
||||
df -h || true
|
||||
free -h || true
|
||||
ps aux --sort=-%mem | head -n 30 || true
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "=== Docker Diagnostics ==="
|
||||
docker version || true
|
||||
docker info || true
|
||||
docker ps -a || true
|
||||
docker images --digests | head -n 50 || true
|
||||
else
|
||||
echo "docker not available on this runner"
|
||||
fi
|
||||
|
||||
echo "=== Kernel Tail ==="
|
||||
dmesg | tail -n 120 || true
|
||||
|
||||
build:
|
||||
name: Build and Push CICD Complete Image
|
||||
# Pin main image build to high-memory worker to reduce setup-time failures.
|
||||
runs-on: ubuntu-act-8gb
|
||||
needs: startup-audit
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
head_sha: ${{ steps.meta.outputs.head_sha }}
|
||||
deployable_backend_tag_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_tag_ref }}
|
||||
deployable_backend_digest_ref: ${{ steps.deployable_backend_ref.outputs.deployable_backend_digest_ref }}
|
||||
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
echo "=== Runner Identity ==="
|
||||
echo "runner_name=${RUNNER_NAME:-}"
|
||||
echo "runner_name_hint=${GITEA_RUNNER_NAME:-${ACT_RUNNER_NAME:-${RUNNER_NAME:-unknown}}}"
|
||||
echo "runner_hostname_env=${HOSTNAME:-unknown}"
|
||||
echo "runner_uname_n=$(uname -n 2>/dev/null || echo unknown)"
|
||||
echo "runner_etc_hostname=$(cat /etc/hostname 2>/dev/null || echo unknown)"
|
||||
echo "runner_os=${RUNNER_OS:-unknown}"
|
||||
echo "runner_arch=${RUNNER_ARCH:-unknown}"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
- name: Audit trigger context
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SOURCE_WORKFLOW: ${{ github.event.inputs.source_workflow }}
|
||||
BASE_NEEDED: ${{ github.event.inputs.base_needed }}
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }}
|
||||
REF: ${{ github.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESOLVED_HEAD_SHA:0:8}}"
|
||||
echo "=== Dispatch Audit: CICD Main Build ==="
|
||||
echo "event_name=${EVENT_NAME}"
|
||||
echo "source_workflow=${SOURCE_WORKFLOW}"
|
||||
echo "base_needed=${BASE_NEEDED}"
|
||||
echo "head_sha_input=${HEAD_SHA_INPUT}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}"
|
||||
echo "base_hash_input=${BASE_HASH_INPUT}"
|
||||
echo "ref=${REF}"
|
||||
echo "ref_name=${REF_NAME}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
|
||||
- name: Resolve head SHA
|
||||
id: meta
|
||||
env:
|
||||
HEAD_SHA_INPUT: ${{ github.event.inputs.head_sha }}
|
||||
HEAD_SHA_FALLBACK: ${{ github.sha }}
|
||||
run: |
|
||||
RESOLVED_HEAD_SHA="${HEAD_SHA_INPUT:-${HEAD_SHA_FALLBACK}}"
|
||||
echo "head_sha=${RESOLVED_HEAD_SHA}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Minimal checkout for build and verification inputs
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
run: |
|
||||
set -e
|
||||
umask 077
|
||||
trap 'rm -f ~/.ssh/id_rsa' EXIT
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
mkdir -p ~/.ssh
|
||||
echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -p "${GITEA_SSH_PORT}" "${GITEA_SSH_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git clone --depth 1 --no-checkout "${GITEA_REPO_SSH_URL}" .
|
||||
|
||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" \
|
||||
git fetch --depth 1 origin "${HEAD_SHA}" >/dev/null 2>&1 || true
|
||||
|
||||
git checkout FETCH_HEAD -- \
|
||||
.dockerignore \
|
||||
Dockerfile.backend \
|
||||
Dockerfile.frontend \
|
||||
Dockerfile.cicd \
|
||||
Dockerfile.cicd-base \
|
||||
backend \
|
||||
frontend \
|
||||
scripts/compute-cicd-base-hash.sh \
|
||||
scripts/check-dockerfile-boundaries.sh \
|
||||
scripts/verify-deployable-image-purity.sh
|
||||
chmod +x scripts/compute-cicd-base-hash.sh
|
||||
|
||||
- name: Verify deployable runtime boundaries
|
||||
run: |
|
||||
set -e
|
||||
bash ./scripts/check-dockerfile-boundaries.sh
|
||||
|
||||
- name: Build and verify deployable runtime image purity
|
||||
env:
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
run: |
|
||||
set -e
|
||||
|
||||
docker build -f Dockerfile.backend \
|
||||
-t deployable-backend:"${HEAD_SHA}" .
|
||||
|
||||
docker build -f Dockerfile.frontend \
|
||||
--target production \
|
||||
-t deployable-frontend:"${HEAD_SHA}" .
|
||||
|
||||
bash ./scripts/verify-deployable-image-purity.sh \
|
||||
--image deployable-backend:"${HEAD_SHA}" \
|
||||
--profile backend
|
||||
|
||||
bash ./scripts/verify-deployable-image-purity.sh \
|
||||
--image deployable-frontend:"${HEAD_SHA}" \
|
||||
--profile frontend
|
||||
|
||||
- name: Push deployable backend runtime image
|
||||
id: deployable_backend_ref
|
||||
env:
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
||||
|
||||
DEPLOYABLE_BACKEND_REPO="${GITEA_REGISTRY}/darkhelm.org/deployable-backend"
|
||||
DEPLOYABLE_BACKEND_TAG_REF="${DEPLOYABLE_BACKEND_REPO}:${HEAD_SHA}"
|
||||
|
||||
docker tag "deployable-backend:${HEAD_SHA}" "${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
docker push "${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
docker pull "${DEPLOYABLE_BACKEND_TAG_REF}" >/dev/null
|
||||
|
||||
DEPLOYABLE_BACKEND_DIGEST_REF="$({
|
||||
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${DEPLOYABLE_BACKEND_TAG_REF}" \
|
||||
| grep "^${DEPLOYABLE_BACKEND_REPO}@sha256:" \
|
||||
| head -n 1
|
||||
} || true)"
|
||||
|
||||
if [ -z "${DEPLOYABLE_BACKEND_DIGEST_REF}" ]; then
|
||||
echo "❌ Unable to resolve deployable backend digest reference"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}" >> "$GITHUB_OUTPUT"
|
||||
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}" >> "$GITHUB_OUTPUT"
|
||||
echo "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
echo "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
||||
|
||||
- name: Build and push complete CICD image
|
||||
env:
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
BASE_HASH_INPUT: ${{ github.event.inputs.base_hash }}
|
||||
run: |
|
||||
set -e
|
||||
umask 077
|
||||
trap 'rm -f /tmp/ssh_key' EXIT
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
||||
|
||||
if [ -n "${BASE_HASH_INPUT}" ]; then
|
||||
BASE_HASH="${BASE_HASH_INPUT}"
|
||||
echo "Using provided immutable base hash from upstream workflow dispatch: ${BASE_HASH}"
|
||||
else
|
||||
BASE_HASH=$(./scripts/compute-cicd-base-hash.sh)
|
||||
echo "No base_hash input provided; computed base hash locally: ${BASE_HASH}"
|
||||
fi
|
||||
|
||||
BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
|
||||
|
||||
verify_base_image() {
|
||||
# Fast path: base image already present locally.
|
||||
if docker image inspect "${BASE_IMAGE}" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Manifest inspect unreliable on these runners; use pull as truth.
|
||||
timeout 1800 docker pull "${BASE_IMAGE}" >/tmp/base-image-pull.log 2>&1
|
||||
}
|
||||
|
||||
max_attempts=2
|
||||
sleep_seconds=15
|
||||
for i in $(seq 1 "${max_attempts}"); do
|
||||
echo "Base availability check ${i}/${max_attempts}..."
|
||||
if verify_base_image; then
|
||||
echo "✓ Base image available and hash matched: ${BASE_IMAGE}"
|
||||
break
|
||||
else
|
||||
pull_exit=$?
|
||||
echo "Base pull attempt ${i} failed with exit code ${pull_exit}"
|
||||
if [ "${pull_exit}" -eq 124 ]; then
|
||||
echo "Pull timed out after 1800s while downloading base image"
|
||||
fi
|
||||
if [ -s /tmp/base-image-pull.log ]; then
|
||||
tail -n 40 /tmp/base-image-pull.log || true
|
||||
fi
|
||||
|
||||
if [ "${i}" -eq "${max_attempts}" ]; then
|
||||
echo "❌ Required immutable base image is not available or mismatched: ${BASE_IMAGE}"
|
||||
exit 1
|
||||
fi
|
||||
sleep "${sleep_seconds}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✓ Base image ready: ${BASE_IMAGE}"
|
||||
|
||||
echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
|
||||
chmod 600 /tmp/ssh_key
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
docker build -f Dockerfile.cicd \
|
||||
--secret id=ssh_private_key,src=/tmp/ssh_key \
|
||||
--add-host "${GITEA_SSH_HOST}:${GITEA_REGISTRY_IP}" \
|
||||
--build-arg GITHUB_SHA="${HEAD_SHA}" \
|
||||
--build-arg CICD_BASE_IMAGE="${BASE_IMAGE}" \
|
||||
-t cicd:latest .
|
||||
|
||||
docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest"
|
||||
docker tag cicd:latest "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
||||
|
||||
docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:latest"
|
||||
docker push "${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd:${HEAD_SHA}"
|
||||
|
||||
- *failure_diagnostics_step
|
||||
|
||||
dispatch-tests:
|
||||
name: Dispatch CICD Tests
|
||||
runs-on: ubuntu-act
|
||||
timeout-minutes: 15
|
||||
needs: build
|
||||
if: needs.build.result == 'success'
|
||||
steps:
|
||||
- name: Dispatch tests workflow
|
||||
env:
|
||||
ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }}
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
HEAD_SHA: ${{ needs.build.outputs.head_sha }}
|
||||
DEPLOYABLE_BACKEND_TAG_REF: ${{ needs.build.outputs.deployable_backend_tag_ref }}
|
||||
DEPLOYABLE_BACKEND_DIGEST_REF: ${{ needs.build.outputs.deployable_backend_digest_ref }}
|
||||
REPO_FULL: ${{ github.repository }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
TRACE_ID_INPUT: ${{ github.event.inputs.trace_id }}
|
||||
run: |
|
||||
set -e
|
||||
|
||||
DISPATCH_TOKEN="${ACTIONS_TRIGGER_TOKEN:-${PACKAGE_ACCESS_TOKEN:-}}"
|
||||
|
||||
if [ -z "${DISPATCH_TOKEN}" ]; then
|
||||
echo "❌ Missing dispatch token. Set ACTIONS_TRIGGER_TOKEN (repo write scope) or ensure PACKAGE_ACCESS_TOKEN has Actions workflow-dispatch permissions."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
REPO_OWNER="${REPO_FULL%/*}"
|
||||
REPO_NAME="${REPO_FULL#*/}"
|
||||
TARGET_REF="${HEAD_REF:-${REF_NAME}}"
|
||||
TRACE_ID="${TRACE_ID_INPUT:-cicd-main-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${HEAD_SHA:0:8}}"
|
||||
|
||||
echo "trace_id=${TRACE_ID}"
|
||||
echo "target_ref=${TARGET_REF}"
|
||||
|
||||
CANDIDATE_API_BASES=()
|
||||
if [ -n "${GITHUB_SERVER_URL:-}" ]; then
|
||||
CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
|
||||
fi
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
|
||||
CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
|
||||
|
||||
ensure_curl() {
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq curl ca-certificates
|
||||
fi
|
||||
command -v curl >/dev/null 2>&1
|
||||
}
|
||||
|
||||
ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; }
|
||||
|
||||
HELPER_PATH="/tmp/dispatch-workflow.sh"
|
||||
|
||||
fetch_dispatch_helper() {
|
||||
local helper_ref="$1"
|
||||
local api_base
|
||||
for api_base in "${CANDIDATE_API_BASES[@]}"; do
|
||||
helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
|
||||
if curl -fsS --connect-timeout 5 --max-time 20 \
|
||||
-H "Authorization: token ${DISPATCH_TOKEN}" \
|
||||
-H "User-Agent: plex-playlist-cicd-main" \
|
||||
-o "${HELPER_PATH}" \
|
||||
"${helper_url}"; then
|
||||
chmod +x "${HELPER_PATH}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
|
||||
echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DISPATCH_ARGS=(
|
||||
--token "${DISPATCH_TOKEN}"
|
||||
--repo "${REPO_FULL}"
|
||||
--workflow "cicd-tests.yaml"
|
||||
--ref "${TARGET_REF}"
|
||||
--head-sha "${HEAD_SHA}"
|
||||
--source-workflow "CICD Main Build"
|
||||
--trace-id "${TRACE_ID}"
|
||||
--input "deployable_backend_tag_ref=${DEPLOYABLE_BACKEND_TAG_REF}"
|
||||
--input "deployable_backend_digest_ref=${DEPLOYABLE_BACKEND_DIGEST_REF}"
|
||||
)
|
||||
|
||||
for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
|
||||
DISPATCH_ARGS+=(--api-base "${API_BASE}")
|
||||
done
|
||||
|
||||
"${HELPER_PATH}" "${DISPATCH_ARGS[@]}"
|
||||
|
||||
- *failure_diagnostics_step
|
||||
@@ -20,213 +20,84 @@ jobs:
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Setup Node.js for Renovate
|
||||
- name: Prepare Renovate container image
|
||||
env:
|
||||
RENOVATE_IMAGE_PRIMARY: kankali.darkhelm.lan:3001/darkhelm.org/renovate:41
|
||||
RENOVATE_IMAGE_FALLBACK: ghcr.io/renovatebot/renovate:41
|
||||
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
|
||||
GITEA_REGISTRY_IP: 10.18.75.2
|
||||
run: |
|
||||
echo "=== Setting up Node.js 24 for Renovate ==="
|
||||
set -euo pipefail
|
||||
|
||||
# Check existing Node.js
|
||||
if command -v node &> /dev/null; then
|
||||
echo "Current Node.js version: $(node --version)"
|
||||
fi
|
||||
if command -v npm &> /dev/null; then
|
||||
echo "Current npm version: $(npm --version)"
|
||||
retry_cmd() {
|
||||
attempts="${1:-3}"
|
||||
backoff="${2:-10}"
|
||||
shift 2
|
||||
|
||||
attempt=1
|
||||
while [ "${attempt}" -le "${attempts}" ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "${attempt}" -lt "${attempts}" ]; then
|
||||
sleep_seconds=$((backoff * attempt))
|
||||
echo "Command failed (attempt ${attempt}/${attempts}): $*" >&2
|
||||
echo "Retrying in ${sleep_seconds}s" >&2
|
||||
sleep "${sleep_seconds}"
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after ${attempts} attempts: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "=== Preparing Renovate container ==="
|
||||
free -h || true
|
||||
|
||||
if ! grep -q "${GITEA_REGISTRY_HOST}" /etc/hosts; then
|
||||
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
|
||||
fi
|
||||
|
||||
# Aggressive cleanup of all Node.js/npm installations
|
||||
echo "Performing complete Node.js cleanup..."
|
||||
pick_renovate_image() {
|
||||
for candidate in "${RENOVATE_IMAGE_PRIMARY}" "${RENOVATE_IMAGE_FALLBACK}"; do
|
||||
echo "Trying Renovate image candidate: ${candidate}" >&2
|
||||
|
||||
# Stop any Node.js processes
|
||||
sudo pkill -f node || true
|
||||
if docker image inspect "${candidate}" >/dev/null 2>&1; then
|
||||
echo "Using cached Renovate image: ${candidate}" >&2
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Remove all package-managed Node.js installations
|
||||
sudo apt-get remove -y --purge nodejs npm node || true
|
||||
sudo apt-get autoremove -y --purge || true
|
||||
if retry_cmd 3 15 docker pull "${candidate}" >/dev/null; then
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Remove all manual installations and caches
|
||||
sudo rm -rf /usr/local/bin/node* /usr/local/bin/npm* || true
|
||||
sudo rm -rf /usr/local/lib/node* /usr/local/include/node* || true
|
||||
sudo rm -rf ~/.npm ~/.nvm ~/.node* || true
|
||||
sudo rm -rf /root/.npm /root/.nvm /root/.node* || true
|
||||
sudo rm -rf /usr/share/nodejs || true
|
||||
sudo rm -rf /etc/apt/sources.list.d/nodesource.list* || true
|
||||
echo "⚠ Renovate image candidate failed: ${candidate}" >&2
|
||||
done
|
||||
|
||||
# Clear npm environment variables that might conflict
|
||||
unset npm_config_prefix npm_config_cache npm_config_globalconfig npm_config_init_module || true
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "✓ Cleanup completed"
|
||||
|
||||
# Install Node.js 24 from NodeSource with error handling
|
||||
echo "Installing Node.js 24..."
|
||||
|
||||
# Remove any existing NodeSource repository
|
||||
sudo rm -f /etc/apt/sources.list.d/nodesource.list || true
|
||||
|
||||
# Add NodeSource repository
|
||||
curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -
|
||||
|
||||
# Install with DEBIAN_FRONTEND to avoid interactive prompts
|
||||
echo "Installing Node.js package..."
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs
|
||||
|
||||
# Verify and fix installation
|
||||
echo "=== Verifying Node.js Installation ==="
|
||||
|
||||
# Check Node.js
|
||||
if command -v node &> /dev/null; then
|
||||
NODE_VERSION=$(node --version)
|
||||
echo "✓ Node.js installed: $NODE_VERSION"
|
||||
else
|
||||
echo "❌ Node.js installation failed"
|
||||
if ! RENOVATE_IMAGE="$(pick_renovate_image)"; then
|
||||
echo "❌ Unable to prepare Renovate image from local mirror or GHCR"
|
||||
echo "Tried: ${RENOVATE_IMAGE_PRIMARY}"
|
||||
echo "Tried: ${RENOVATE_IMAGE_FALLBACK}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check npm and fix if needed
|
||||
if command -v npm &> /dev/null && npm --version &> /dev/null; then
|
||||
NPM_VERSION=$(npm --version)
|
||||
echo "✓ npm working: $NPM_VERSION"
|
||||
else
|
||||
echo "⚠️ npm not working properly, reinstalling..."
|
||||
echo "Using Renovate image: ${RENOVATE_IMAGE}"
|
||||
|
||||
# Method 1: Try to fix npm with the bundled version
|
||||
if [ -f "/usr/bin/node" ] && [ -f "/usr/lib/node_modules/npm/bin/npm-cli.js" ]; then
|
||||
echo "Using bundled npm..."
|
||||
sudo ln -sf /usr/lib/node_modules/npm/bin/npm-cli.js /usr/bin/npm || true
|
||||
sudo chmod +x /usr/bin/npm || true
|
||||
fi
|
||||
docker run --rm "${RENOVATE_IMAGE}" --version
|
||||
|
||||
# Method 2: If that doesn't work, reinstall npm manually
|
||||
if ! npm --version &> /dev/null; then
|
||||
echo "Manual npm installation..."
|
||||
curl -L https://www.npmjs.com/install.sh | sudo sh
|
||||
fi
|
||||
|
||||
# Method 3: Last resort - use npx to bootstrap npm
|
||||
if ! npm --version &> /dev/null; then
|
||||
echo "Using node to run npm directly..."
|
||||
# Create npm wrapper script
|
||||
echo '#!/bin/bash' | sudo tee /usr/bin/npm > /dev/null
|
||||
echo 'exec /usr/bin/node /usr/lib/node_modules/npm/bin/npm-cli.js "$@"' | sudo tee -a /usr/bin/npm > /dev/null
|
||||
sudo chmod +x /usr/bin/npm
|
||||
fi
|
||||
|
||||
# Final verification
|
||||
if npm --version &> /dev/null; then
|
||||
echo "✓ npm recovered successfully: $(npm --version)"
|
||||
else
|
||||
echo "❌ npm recovery failed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test npm basic functionality
|
||||
echo "Testing npm functionality..."
|
||||
if npm config get registry &> /dev/null; then
|
||||
echo "✓ npm configuration accessible"
|
||||
else
|
||||
echo "⚠️ npm configuration issues, but continuing..."
|
||||
fi
|
||||
|
||||
# Check version compatibility for Renovate
|
||||
NODE_VERSION=$(node --version | cut -d'v' -f2)
|
||||
echo "=== Version Compatibility Check ==="
|
||||
echo "Node.js version: $NODE_VERSION"
|
||||
|
||||
if [[ $(echo "$NODE_VERSION 24.10.0" | awk '{print ($1 >= $2)}') == 1 ]]; then
|
||||
echo "✅ Node.js version $NODE_VERSION meets Renovate latest requirements"
|
||||
echo "RENOVATE_VERSION=latest" >> $GITHUB_ENV
|
||||
else
|
||||
echo "⚠️ Node.js version $NODE_VERSION - will use compatible Renovate version"
|
||||
echo "RENOVATE_VERSION=40.3.2" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Install Renovate
|
||||
run: |
|
||||
echo "=== Installing Renovate ==="
|
||||
|
||||
# Set npm configuration for memory efficiency
|
||||
npm config set fund false
|
||||
npm config set audit false
|
||||
npm config set progress false
|
||||
npm config set maxsockets 3
|
||||
npm config set fetch-retry-mintimeout 2000
|
||||
npm config set fetch-retry-maxtimeout 10000
|
||||
|
||||
# Use the version determined in previous step
|
||||
echo "Installing Renovate version: $RENOVATE_VERSION"
|
||||
|
||||
# Check available memory
|
||||
echo "Memory info:"
|
||||
free -h || echo "free command not available"
|
||||
|
||||
# Install with retry logic and memory-efficient options
|
||||
for i in 1 2 3; do
|
||||
echo "Renovate installation attempt $i/3..."
|
||||
|
||||
# Clear npm cache and temp files
|
||||
npm cache clean --force || true
|
||||
rm -rf /tmp/npm-* || true
|
||||
|
||||
# Use memory-efficient installation with longer timeout
|
||||
if NODE_OPTIONS="--max-old-space-size=2048" timeout 600 npm install -g "renovate@$RENOVATE_VERSION" \
|
||||
--no-audit \
|
||||
--no-fund \
|
||||
--no-optional \
|
||||
--production \
|
||||
--maxsockets=3 \
|
||||
--fetch-timeout=30000; then
|
||||
echo "✓ Renovate installation successful on attempt $i"
|
||||
break
|
||||
else
|
||||
exit_code=$?
|
||||
echo "⚠️ Renovate installation attempt $i failed (exit code: $exit_code)"
|
||||
|
||||
# Check if it was killed due to memory
|
||||
if [ $exit_code -eq 137 ] || [ $exit_code -eq 143 ]; then
|
||||
echo " Process was killed (likely out of memory)"
|
||||
fi
|
||||
|
||||
if [ $i -eq 3 ]; then
|
||||
echo "❌ All Renovate installation attempts failed"
|
||||
echo "Debugging information:"
|
||||
echo "Node.js version: $(node --version)"
|
||||
echo "npm version: $(npm --version)"
|
||||
echo "Memory status:"
|
||||
free -h || echo "Memory info unavailable"
|
||||
echo "npm config:"
|
||||
npm config list || echo "npm config failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting 30 seconds before retry..."
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
# Verify Renovate installation
|
||||
echo "✓ Renovate version: $(renovate --version)"
|
||||
echo "✓ Renovate location: $(which renovate)"
|
||||
echo "RENOVATE_IMAGE=${RENOVATE_IMAGE}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Configure Renovate for Gitea
|
||||
run: |
|
||||
echo "=== Configuring Renovate for Gitea ==="
|
||||
|
||||
# Renovate reads RENOVATE_TOKEN, RENOVATE_PLATFORM, RENOVATE_ENDPOINT,
|
||||
# and RENOVATE_GIT_AUTHOR directly from the environment at runtime.
|
||||
# Keeping the config file minimal avoids repeated migration warnings.
|
||||
cat > renovate-config.js << 'EOF'
|
||||
module.exports = {
|
||||
platform: 'gitea',
|
||||
endpoint: 'https://dogar.darkhelm.org',
|
||||
gitAuthor: 'Renovate Bot <renovate@darkhelm.org>',
|
||||
repositories: ['DarkHelm.org/plex-playlist'],
|
||||
onboarding: false,
|
||||
requireConfig: 'required',
|
||||
extends: ['local>DarkHelm.org/plex-playlist'],
|
||||
prConcurrentLimit: 3,
|
||||
branchConcurrentLimit: 5,
|
||||
};
|
||||
EOF
|
||||
|
||||
echo "✓ Renovate configuration created"
|
||||
echo "✓ Renovate runtime configuration will be passed through environment and CLI flags"
|
||||
|
||||
- name: Run Renovate
|
||||
env:
|
||||
@@ -235,28 +106,28 @@ jobs:
|
||||
ACTIONS_TRIGGER_TOKEN: ${{ secrets.ACTIONS_TRIGGER_TOKEN }}
|
||||
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
|
||||
RENOVATE_DRY_RUN: ${{ inputs.dry_run }}
|
||||
RENOVATE_CONFIG_FILE: renovate-config.js
|
||||
RENOVATE_PLATFORM: gitea
|
||||
RENOVATE_ENDPOINT: https://dogar.darkhelm.org
|
||||
RENOVATE_ENDPOINT_INTERNAL: http://kankali.darkhelm.lan:3001/api/v1
|
||||
RENOVATE_ENDPOINT_EXTERNAL: https://dogar.darkhelm.org/api/v1
|
||||
RENOVATE_GIT_HOST: dogar.darkhelm.org
|
||||
RENOVATE_GIT_USERNAME: ${{ github.actor }}
|
||||
RENOVATE_ENDPOINT_IP: 10.18.75.2
|
||||
RENOVATE_ALLOW_INSECURE_TLS: "true"
|
||||
LOG_LEVEL: info
|
||||
RENOVATE_GIT_AUTHOR: Renovate Bot <renovate@darkhelm.org>
|
||||
RENOVATE_ONBOARDING: "false"
|
||||
RENOVATE_REQUIRE_CONFIG: required
|
||||
RENOVATE_PR_CONCURRENT_LIMIT: "3"
|
||||
RENOVATE_BRANCH_CONCURRENT_LIMIT: "5"
|
||||
RENOVATE_NODE_ARGS: --max-old-space-size=768
|
||||
RENOVATE_OSV_VULNERABILITY_ALERTS: "false"
|
||||
LOG_LEVEL: debug
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== Running Renovate Bot ==="
|
||||
TARGET_REPO="DarkHelm.org/plex-playlist"
|
||||
TARGET_ORG="${TARGET_REPO%%/*}"
|
||||
|
||||
RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT%/}"
|
||||
if [[ "${RENOVATE_ENDPOINT_EFFECTIVE}" == */api/v1 ]]; then
|
||||
API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}"
|
||||
RENOVATE_ENDPOINT_EFFECTIVE="${RENOVATE_ENDPOINT_EFFECTIVE%/api/v1}"
|
||||
else
|
||||
API_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}/api/v1"
|
||||
fi
|
||||
|
||||
export RENOVATE_ENDPOINT="${RENOVATE_ENDPOINT_EFFECTIVE}"
|
||||
echo "Renovate endpoint: ${RENOVATE_ENDPOINT}"
|
||||
echo "Preflight API endpoint: ${API_ENDPOINT}"
|
||||
|
||||
CURL_INSECURE_FLAG=""
|
||||
if [ "${RENOVATE_ALLOW_INSECURE_TLS:-false}" = "true" ]; then
|
||||
echo "⚠ Renovate insecure TLS mode enabled for self-signed certificate endpoint"
|
||||
@@ -264,6 +135,101 @@ jobs:
|
||||
export NODE_TLS_REJECT_UNAUTHORIZED=0
|
||||
fi
|
||||
|
||||
configure_endpoint() {
|
||||
endpoint_input="${1%/}"
|
||||
endpoint_ip_fallback="${2:-}"
|
||||
|
||||
if [[ "${endpoint_input}" == */api/v1 ]]; then
|
||||
API_ENDPOINT="${endpoint_input}"
|
||||
BASE_ENDPOINT="${endpoint_input%/api/v1}"
|
||||
else
|
||||
API_ENDPOINT="${endpoint_input}/api/v1"
|
||||
BASE_ENDPOINT="${endpoint_input}"
|
||||
fi
|
||||
|
||||
ENDPOINT_AUTHORITY="$(printf '%s' "${BASE_ENDPOINT}" | sed -E 's#^https?://([^/]+).*$#\1#')"
|
||||
ENDPOINT_HOST="${ENDPOINT_AUTHORITY%%:*}"
|
||||
ENDPOINT_IP="$(getent hosts "${ENDPOINT_HOST}" | awk '{print $1; exit}' || true)"
|
||||
if [ -z "${ENDPOINT_IP}" ] && [ -n "${endpoint_ip_fallback}" ]; then
|
||||
ENDPOINT_IP="${endpoint_ip_fallback}"
|
||||
echo "Using configured Renovate endpoint IP fallback: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}"
|
||||
if ! grep -q "${ENDPOINT_HOST}" /etc/hosts; then
|
||||
echo "${ENDPOINT_IP} ${ENDPOINT_HOST}" >> /etc/hosts
|
||||
fi
|
||||
fi
|
||||
|
||||
DOCKER_ADD_HOST_ARG=""
|
||||
if [ -n "${ENDPOINT_IP}" ]; then
|
||||
DOCKER_ADD_HOST_ARG="--add-host ${ENDPOINT_HOST}:${ENDPOINT_IP}"
|
||||
echo "Renovate container host mapping: ${ENDPOINT_HOST} -> ${ENDPOINT_IP}"
|
||||
else
|
||||
echo "⚠ Unable to resolve ${ENDPOINT_HOST} on runner; proceeding without explicit --add-host"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
select_reachable_endpoint() {
|
||||
for candidate in "${RENOVATE_ENDPOINT_INTERNAL}" "${RENOVATE_ENDPOINT_EXTERNAL}"; do
|
||||
fallback_ip=""
|
||||
if [ "${candidate}" = "${RENOVATE_ENDPOINT_INTERNAL}" ]; then
|
||||
fallback_ip="${RENOVATE_ENDPOINT_IP:-}"
|
||||
fi
|
||||
|
||||
configure_endpoint "${candidate}" "${fallback_ip}"
|
||||
echo "Testing Renovate endpoint candidate: ${API_ENDPOINT}"
|
||||
|
||||
version_status=$(curl -sS -o /tmp/renovate-version-check.json -w "%{http_code}" \
|
||||
${CURL_INSECURE_FLAG} \
|
||||
"${API_ENDPOINT}/version" || true)
|
||||
|
||||
if [ "${version_status}" = "200" ]; then
|
||||
SELECTED_ENDPOINT_INPUT="${candidate}"
|
||||
VERSION_STATUS="${version_status}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "⚠ Endpoint candidate failed version preflight: ${API_ENDPOINT} (status=${version_status})"
|
||||
if [ -s /tmp/renovate-version-check.json ]; then
|
||||
cat /tmp/renovate-version-check.json || true
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! select_reachable_endpoint; then
|
||||
echo "❌ Unable to reach either configured Renovate endpoint"
|
||||
echo "Tried internal: ${RENOVATE_ENDPOINT_INTERNAL}"
|
||||
echo "Tried external: ${RENOVATE_ENDPOINT_EXTERNAL}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Force git clone base URL to the selected endpoint host/port.
|
||||
# This avoids relying on platform-advertised clone URLs that may point to an unreachable external proxy.
|
||||
export RENOVATE_GIT_URL="${BASE_ENDPOINT}/"
|
||||
export RENOVATE_ENDPOINT="${BASE_ENDPOINT}"
|
||||
echo "Renovate endpoint (primary): ${RENOVATE_ENDPOINT}"
|
||||
echo "Renovate endpoint (fallback): ${RENOVATE_ENDPOINT_EXTERNAL}"
|
||||
echo "Renovate git base URL override: ${RENOVATE_GIT_URL}"
|
||||
echo "Preflight API endpoint: ${API_ENDPOINT}"
|
||||
|
||||
PLATFORM_VERSION=""
|
||||
if [ "${VERSION_STATUS}" = "200" ]; then
|
||||
PLATFORM_VERSION=$(sed -n 's/.*"version":"\([^"]*\)".*/\1/p' /tmp/renovate-version-check.json | head -n 1)
|
||||
if [ -n "${PLATFORM_VERSION}" ]; then
|
||||
export RENOVATE_X_PLATFORM_VERSION="${PLATFORM_VERSION}"
|
||||
echo "✓ Gitea version preflight passed: ${RENOVATE_X_PLATFORM_VERSION}"
|
||||
else
|
||||
echo "⚠ Gitea version preflight returned 200 but version could not be parsed"
|
||||
cat /tmp/renovate-version-check.json || true
|
||||
fi
|
||||
else
|
||||
echo "⚠ Gitea version preflight failed with status ${VERSION_STATUS}"
|
||||
if [ -s /tmp/renovate-version-check.json ]; then
|
||||
cat /tmp/renovate-version-check.json || true
|
||||
fi
|
||||
fi
|
||||
|
||||
select_token_with_repo_access() {
|
||||
for candidate_name in RENOVATE_TOKEN_SECRET ACTIONS_TRIGGER_TOKEN PACKAGE_ACCESS_TOKEN; do
|
||||
candidate_value="${!candidate_name:-}"
|
||||
@@ -286,7 +252,10 @@ jobs:
|
||||
-H "Authorization: token ${candidate_value}" \
|
||||
"${API_ENDPOINT}/orgs/${TARGET_ORG}" || true)
|
||||
|
||||
if [ "${USER_STATUS}" = "200" ] && [ "${REPO_STATUS}" = "200" ] && [ "${ORG_STATUS}" = "200" ]; then
|
||||
if [ "${REPO_STATUS}" = "200" ]; then
|
||||
if [ "${USER_STATUS}" != "200" ] || [ "${ORG_STATUS}" != "200" ]; then
|
||||
echo "⚠ Token candidate ${candidate_name} has repo access but limited user/org scopes (user=${USER_STATUS}, repo=${REPO_STATUS}, org=${ORG_STATUS})"
|
||||
fi
|
||||
echo "${candidate_name}:${candidate_value}"
|
||||
return 0
|
||||
fi
|
||||
@@ -299,7 +268,7 @@ jobs:
|
||||
|
||||
if ! SELECTED_TOKEN_RESULT="$(select_token_with_repo_access)"; then
|
||||
echo "❌ No token available for Renovate authentication with repository access"
|
||||
echo "Configure RENOVATE_TOKEN with repo+issue write and organization/user read scopes."
|
||||
echo "Configure RENOVATE_TOKEN with at least repository read/write access; user/org read scopes are optional for this workflow."
|
||||
echo "Token preflight checks attempted: ${API_ENDPOINT}/user, ${API_ENDPOINT}/repos/${TARGET_REPO}, and ${API_ENDPOINT}/orgs/${TARGET_ORG}"
|
||||
if [ -s /tmp/renovate-auth-check-user.json ]; then
|
||||
echo "Last user endpoint response body:"
|
||||
@@ -319,10 +288,37 @@ jobs:
|
||||
SELECTED_TOKEN_SOURCE="${SELECTED_TOKEN_RESULT%%:*}"
|
||||
SELECTED_TOKEN="${SELECTED_TOKEN_RESULT#*:}"
|
||||
export RENOVATE_TOKEN="${SELECTED_TOKEN}"
|
||||
export RENOVATE_HOST_RULES="[{\"matchHost\":\"${ENDPOINT_AUTHORITY}\",\"token\":\"${SELECTED_TOKEN}\",\"authType\":\"Token-Only\"}]"
|
||||
# Rewrite external clone URLs to local-network git transport.
|
||||
export RENOVATE_GIT_REWRITE_FROM="https://dogar.darkhelm.org/"
|
||||
export RENOVATE_GIT_REWRITE_TO="http://${RENOVATE_GIT_USERNAME}:${SELECTED_TOKEN}@kankali.darkhelm.lan:3001/"
|
||||
|
||||
# Prefer local SSH for git on kankali:2222 when keys are available in the container.
|
||||
if docker run --rm ${DOCKER_ADD_HOST_ARG} \
|
||||
--entrypoint /bin/sh \
|
||||
"${RENOVATE_IMAGE}" \
|
||||
-lc 'GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5" git ls-remote --heads "ssh://git@kankali.darkhelm.lan:2222/'"${TARGET_REPO}"'.git" >/dev/null 2>&1'; then
|
||||
export RENOVATE_GIT_REWRITE_TO="ssh://git@kankali.darkhelm.lan:2222/"
|
||||
echo "✓ Renovate git rewrite transport: local SSH (kankali:2222)"
|
||||
else
|
||||
echo "⚠ Local SSH git auth unavailable in Renovate container; using internal HTTP token fallback"
|
||||
fi
|
||||
unset SELECTED_TOKEN RESULT_TOKEN
|
||||
|
||||
echo "✓ Renovate auth preflight passed with ${SELECTED_TOKEN_SOURCE}"
|
||||
|
||||
echo "=== Renovate container connectivity preflight ==="
|
||||
# Prove whether the same image/container path Renovate uses can reach Gitea.
|
||||
# If this fails, the problem is container network/TLS resolution, not Renovate config.
|
||||
# shellcheck disable=SC2086
|
||||
docker run --rm ${DOCKER_ADD_HOST_ARG} \
|
||||
-e API_ENDPOINT="${API_ENDPOINT}" \
|
||||
-e TARGET_REPO="${TARGET_REPO}" \
|
||||
-e RENOVATE_TOKEN \
|
||||
-e NODE_TLS_REJECT_UNAUTHORIZED \
|
||||
"${RENOVATE_IMAGE}" \
|
||||
node -e 'const endpoints = [["version", `${process.env.API_ENDPOINT}/version`, {}], ["repo", `${process.env.API_ENDPOINT}/repos/${process.env.TARGET_REPO}`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` } }], ["user", `${process.env.API_ENDPOINT}/user`, { headers: { Authorization: `token ${process.env.RENOVATE_TOKEN}` }, optional: true }]]; let failed = false; (async () => { for (const [name, url, rawOptions] of endpoints) { const optional = Boolean(rawOptions.optional); const options = { ...rawOptions }; delete options.optional; try { const response = await fetch(url, options); console.log(`container_${name}_status=${response.status}`); if (!response.ok && !optional) { failed = true; console.log(`container_${name}_body_start`); console.log((await response.text()).slice(0, 1000)); console.log(`container_${name}_body_end`); } else if (!response.ok && optional) { console.log(`container_${name}_optional_scope_missing=true`); } } catch (error) { if (!optional) { failed = true; console.log(`container_${name}_error=${error instanceof Error ? error.message : String(error)}`); } else { console.log(`container_${name}_optional_probe_error=${error instanceof Error ? error.message : String(error)}`); } } } if (failed) { process.exit(1); } })();'
|
||||
|
||||
# Run Renovate with configuration
|
||||
if [ "${RENOVATE_DRY_RUN}" = "true" ]; then
|
||||
export RENOVATE_DRY_RUN="full"
|
||||
@@ -331,7 +327,58 @@ jobs:
|
||||
unset RENOVATE_DRY_RUN
|
||||
fi
|
||||
|
||||
renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" DarkHelm.org/plex-playlist
|
||||
run_renovate_with_endpoint() {
|
||||
endpoint="$1"
|
||||
export RENOVATE_ENDPOINT="${endpoint}"
|
||||
echo "Running Renovate with endpoint: ${RENOVATE_ENDPOINT}"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
docker run --rm ${DOCKER_ADD_HOST_ARG} \
|
||||
-e LOG_LEVEL \
|
||||
-e NODE_TLS_REJECT_UNAUTHORIZED \
|
||||
-e GIT_SSL_NO_VERIFY=1 \
|
||||
-e GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no' \
|
||||
-e RENOVATE_TOKEN \
|
||||
-e RENOVATE_GIT_USERNAME \
|
||||
-e RENOVATE_GIT_REWRITE_FROM \
|
||||
-e RENOVATE_GIT_REWRITE_TO \
|
||||
-e RENOVATE_GIT_URL \
|
||||
-e RENOVATE_HOST_RULES \
|
||||
-e RENOVATE_X_PLATFORM_VERSION \
|
||||
-e RENOVATE_PLATFORM \
|
||||
-e RENOVATE_ENDPOINT \
|
||||
-e RENOVATE_DRY_RUN \
|
||||
-e RENOVATE_GIT_AUTHOR \
|
||||
-e RENOVATE_ONBOARDING \
|
||||
-e RENOVATE_REQUIRE_CONFIG \
|
||||
-e RENOVATE_PR_CONCURRENT_LIMIT \
|
||||
-e RENOVATE_BRANCH_CONCURRENT_LIMIT \
|
||||
-e RENOVATE_NODE_ARGS \
|
||||
-e RENOVATE_OSV_VULNERABILITY_ALERTS \
|
||||
--entrypoint /bin/sh \
|
||||
"${RENOVATE_IMAGE}" \
|
||||
-lc 'set -e
|
||||
git config --global "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://dogar.darkhelm.org/"
|
||||
git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "http://dogar.darkhelm.org/"
|
||||
git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://${RENOVATE_GIT_USERNAME}@dogar.darkhelm.org/"
|
||||
git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://${RENOVATE_TOKEN}@dogar.darkhelm.org/"
|
||||
git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "https://x-access-token:${RENOVATE_TOKEN}@dogar.darkhelm.org/"
|
||||
git config --global --add "url.${RENOVATE_GIT_REWRITE_TO}.insteadOf" "${RENOVATE_GIT_REWRITE_FROM}"
|
||||
echo "Configured git dogar->kankali rewrite rules"
|
||||
renovate --platform "${RENOVATE_PLATFORM}" --endpoint "${RENOVATE_ENDPOINT}" --git-url "${RENOVATE_GIT_URL}" --git-author "${RENOVATE_GIT_AUTHOR}" --onboarding="${RENOVATE_ONBOARDING}" --pr-concurrent-limit "${RENOVATE_PR_CONCURRENT_LIMIT}" --branch-concurrent-limit "${RENOVATE_BRANCH_CONCURRENT_LIMIT}" --osv-vulnerability-alerts="${RENOVATE_OSV_VULNERABILITY_ALERTS}" DarkHelm.org/plex-playlist'
|
||||
}
|
||||
|
||||
if run_renovate_with_endpoint "${BASE_ENDPOINT}" 2>&1 | tee /tmp/renovate.log; then
|
||||
:
|
||||
else
|
||||
primary_status=$?
|
||||
if [ "${primary_status}" -eq 137 ] || grep -qE '(^|[[:space:]])Killed($|[[:space:]])|exitcode '\''137'\''' /tmp/renovate.log; then
|
||||
echo "❌ Renovate terminated by OOM/kill signal on primary endpoint; skipping endpoint retry"
|
||||
exit "${primary_status}"
|
||||
fi
|
||||
echo "⚠ Renovate run failed with primary endpoint. Retrying with API endpoint form..."
|
||||
run_renovate_with_endpoint "${API_ENDPOINT}" 2>&1 | tee /tmp/renovate.log
|
||||
fi
|
||||
|
||||
echo "✓ Renovate execution completed"
|
||||
|
||||
|
||||
@@ -1,37 +1,30 @@
|
||||
# Backend Dockerfile for FastAPI with Python 3.14
|
||||
FROM python:3.14-slim
|
||||
FROM python:3.14-slim AS dependencies
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
|
||||
|
||||
# Create and activate virtual environment
|
||||
ENV VIRTUAL_ENV=/app/.venv
|
||||
RUN uv venv $VIRTUAL_ENV
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
# Copy dependency files first for better caching
|
||||
COPY backend/pyproject.toml backend/uv.lock* ./
|
||||
# Hatchling resolves the readme path ../README.md relative to the backend
|
||||
# package root (/app). Create a stub so metadata validation succeeds without
|
||||
# requiring the file to pass through .dockerignore.
|
||||
RUN echo '# plex-playlist' > /README.md
|
||||
RUN uv venv "$VIRTUAL_ENV" && uv sync --frozen --no-dev
|
||||
|
||||
# Install runtime dependencies only
|
||||
RUN uv sync --frozen --no-dev
|
||||
FROM python:3.14-slim AS runtime
|
||||
|
||||
# Copy application code
|
||||
COPY backend/ .
|
||||
WORKDIR /app
|
||||
|
||||
ENV VIRTUAL_ENV=/app/.venv
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
COPY --from=dependencies /app/.venv /app/.venv
|
||||
COPY backend/src ./src
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
# Default command - can be overridden in compose for development
|
||||
CMD ["uvicorn", "backend.main:app", "--app-dir", "/app/src", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
8
Dockerfile.e2e-tester
Normal file
8
Dockerfile.e2e-tester
Normal file
@@ -0,0 +1,8 @@
|
||||
ARG PLAYWRIGHT_BASE_IMAGE=mcr.microsoft/playwright:v1.56.1-jammy
|
||||
FROM ${PLAYWRIGHT_BASE_IMAGE}
|
||||
|
||||
WORKDIR /workspace/frontend
|
||||
|
||||
RUN corepack enable
|
||||
|
||||
CMD ["bash"]
|
||||
@@ -1,40 +1,38 @@
|
||||
# Frontend Dockerfile for Vue/Vite TypeScript project
|
||||
FROM node:20-alpine AS base
|
||||
FROM node:24-alpine AS deps
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files first for better caching
|
||||
COPY frontend/package*.json ./
|
||||
COPY frontend/.yarnrc.yml ./
|
||||
COPY frontend/yarn.lock* frontend/pnpm-lock.yaml* ./
|
||||
COPY frontend/package.json frontend/yarn.lock frontend/.yarnrc.yml ./
|
||||
RUN set -eux; \
|
||||
retry() { \
|
||||
attempts="$1"; shift; \
|
||||
n=1; \
|
||||
while [ "$n" -le "$attempts" ]; do \
|
||||
if "$@"; then \
|
||||
return 0; \
|
||||
fi; \
|
||||
if [ "$n" -lt "$attempts" ]; then \
|
||||
sleep "$((3 * n))"; \
|
||||
fi; \
|
||||
n=$((n + 1)); \
|
||||
done; \
|
||||
return 1; \
|
||||
}; \
|
||||
corepack enable; \
|
||||
retry 5 corepack prepare yarn@4.10.3 --activate; \
|
||||
retry 5 yarn install --immutable
|
||||
|
||||
# Install dependencies
|
||||
RUN if [ -f yarn.lock ]; then corepack enable && corepack yarn install; \
|
||||
elif [ -f pnpm-lock.yaml ]; then npm install -g pnpm && pnpm install; \
|
||||
else npm install; fi
|
||||
FROM deps AS build
|
||||
|
||||
# Copy application code
|
||||
COPY frontend/ .
|
||||
RUN yarn build
|
||||
|
||||
# Development stage
|
||||
FROM base AS development
|
||||
EXPOSE 5173
|
||||
CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"]
|
||||
|
||||
# Production build stage
|
||||
FROM base AS build
|
||||
RUN if [ -f yarn.lock ]; then corepack yarn build; \
|
||||
elif [ -f pnpm-lock.yaml ]; then pnpm build; \
|
||||
else npm run build; fi
|
||||
|
||||
# Production stage
|
||||
FROM nginx:alpine AS production
|
||||
# Copy built assets from build stage
|
||||
|
||||
COPY --from=build /app/dist /usr/share/nginx/html
|
||||
# Copy nginx configuration
|
||||
COPY frontend/nginx.conf /etc/nginx/nginx.conf
|
||||
# Expose port
|
||||
|
||||
EXPOSE 80
|
||||
# Start nginx
|
||||
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
|
||||
9
Dockerfile.integration-tester
Normal file
9
Dockerfile.integration-tester
Normal file
@@ -0,0 +1,9 @@
|
||||
FROM python:3.14-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
CMD ["bash"]
|
||||
@@ -157,9 +157,14 @@ plex-playlist/
|
||||
For Gitea Actions runner image mirror maintenance, use:
|
||||
|
||||
- `source scripts/gitea-actions/repair_runner_mirror.xsh`
|
||||
- `source scripts/gitea-actions/check_runner_images.xsh`
|
||||
- `source scripts/gitea-actions/check_runner_images.xsh` (checks ubuntu:22.04, GHCR act image, local mirror, and Renovate image)
|
||||
- `source scripts/gitea-actions/collect_runner_diagnostics.xsh [since] [trace_id]`
|
||||
|
||||
For setup-stage failures with sparse logs, use:
|
||||
|
||||
- `source scripts/gitea-actions/runner-prechange-capture.xsh [tail_lines]`
|
||||
- `source scripts/gitea-actions/diagnose_runner_startup.xsh`
|
||||
|
||||
For full troubleshooting context, see `docs/GITEA_ACTIONS_TROUBLESHOOTING.md`.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
@@ -154,6 +154,8 @@ reset = [
|
||||
{shell = "cd frontend && rm -rf node_modules 2>/dev/null || true"},
|
||||
"deps-install"
|
||||
]
|
||||
runner-diagnostics = {shell = "xonsh ./scripts/gitea-actions/collect_runner_diagnostics.xsh", help = "Collect recent runner, docker, and system diagnostics from all known runner hosts"}
|
||||
runner-discover = {shell = "xonsh ./scripts/gitea-actions/discover-runners.xsh --batch", help = "Collect runner inventory, labels, and baseline health from all known runner hosts"}
|
||||
# === Development Setup (New Developer Onboarding) ===
|
||||
setup = [
|
||||
"deps-install",
|
||||
@@ -166,15 +168,15 @@ setup = [
|
||||
]
|
||||
test-all = ["test-backend", "test-frontend", "test-integration"]
|
||||
# === Testing Tasks ===
|
||||
test-backend = {shell = "cd backend && uv run pytest", help = "Run backend unit tests"}
|
||||
test-backend-cov = {shell = "cd backend && uv run pytest --cov", help = "Run backend tests with coverage"}
|
||||
test-e2e = {shell = "cd frontend && yarn test:e2e", help = "Run end-to-end tests"}
|
||||
test-backend = {shell = "cd backend && uv run pytest -m 'not integration'", help = "Run backend unit tests (excludes integration marker)"}
|
||||
test-backend-cov = {shell = "cd backend && uv run pytest -m 'not integration' --cov", help = "Run backend unit tests with coverage (excludes integration marker)"}
|
||||
test-e2e = {shell = "bash ./scripts/run-ci-e2e-compose.sh", help = "Run E2E tests via compose against deployable runtime images"}
|
||||
test-frontend = {shell = "cd frontend && yarn test", help = "Run frontend unit tests"}
|
||||
test-frontend-cov = {shell = "cd frontend && yarn test:coverage", help = "Run frontend tests with coverage"}
|
||||
test-full = ["test-backend-cov", "test-frontend-cov", "test-integration", "test-e2e"]
|
||||
# === Smart Conditional Tasks ===
|
||||
test-if-changed = {shell = "if git diff --quiet HEAD~1 backend/ frontend/; then echo 'No changes detected, skipping tests'; else poe test-unit; fi", help = "Only run tests if code has changed"}
|
||||
test-integration = {shell = "cd backend && uv run pytest tests/integration/", help = "Run backend integration tests"}
|
||||
test-integration = {shell = "bash ./scripts/run-ci-integration-compose.sh", help = "Run integration tests via compose against deployable runtime image"}
|
||||
test-parallel = {shell = "poe test-backend & poe test-frontend & wait", help = "Run unit tests in parallel for speed"}
|
||||
# === Comprehensive Testing ===
|
||||
test-unit = ["test-backend", "test-frontend"]
|
||||
|
||||
@@ -14,7 +14,6 @@ from backend.main import app, compatibility_status, get_api_session
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestAPIIntegration:
|
||||
"""Integration tests for API endpoints."""
|
||||
|
||||
|
||||
53
backend/tests/integration/test_runtime_blackbox.py
Normal file
53
backend/tests/integration/test_runtime_blackbox.py
Normal file
@@ -0,0 +1,53 @@
|
||||
"""Runtime black-box integration tests executed over the container network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
BACKEND_BASE_URL = os.getenv("INTEGRATION_BACKEND_URL", "http://backend:8000").rstrip(
|
||||
"/"
|
||||
)
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _fetch(path: str) -> tuple[int, str]:
|
||||
"""Fetch an endpoint and return status code and body text."""
|
||||
url = f"{BACKEND_BASE_URL}{path}"
|
||||
request = urllib.request.Request(url=url, method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
return response.status, body
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
return exc.code, body
|
||||
|
||||
|
||||
def test_root_endpoint_runtime() -> None:
|
||||
"""Runtime root endpoint should return backend API message."""
|
||||
status, body = _fetch("/")
|
||||
assert status == 200
|
||||
payload = json.loads(body)
|
||||
assert payload == {"message": "Plex Playlist Backend API"}
|
||||
|
||||
|
||||
def test_health_endpoint_runtime() -> None:
|
||||
"""Runtime health endpoint should report healthy database connectivity."""
|
||||
status, body = _fetch("/health")
|
||||
assert status == 200
|
||||
payload = json.loads(body)
|
||||
assert payload.get("status") == "healthy"
|
||||
assert payload.get("database") == "connected"
|
||||
|
||||
|
||||
def test_compatibility_endpoint_runtime() -> None:
|
||||
"""Runtime compatibility endpoint should report policy check success."""
|
||||
status, body = _fetch("/compatibility")
|
||||
assert status == 200
|
||||
payload = json.loads(body)
|
||||
assert payload.get("ok") is True
|
||||
BIN
cicd-workflow.ods
Normal file
BIN
cicd-workflow.ods
Normal file
Binary file not shown.
28
compose.ci.e2e.yml
Normal file
28
compose.ci.e2e.yml
Normal file
@@ -0,0 +1,28 @@
|
||||
---
|
||||
services:
|
||||
database:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_DB: plex_playlist
|
||||
POSTGRES_USER: plex_user
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-plex_password}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
|
||||
backend:
|
||||
image: ${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}
|
||||
environment:
|
||||
DATABASE_URL: ${DB_URL:-postgresql://plex_user:plex_password@database:5432/plex_playlist}
|
||||
ENVIRONMENT: production
|
||||
depends_on:
|
||||
database:
|
||||
condition: service_healthy
|
||||
|
||||
frontend:
|
||||
image: ${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local}
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_started
|
||||
22
compose.ci.integration.yml
Normal file
22
compose.ci.integration.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
---
|
||||
services:
|
||||
database:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_DB: plex_playlist
|
||||
POSTGRES_USER: plex_user
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-plex_password}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U plex_user -d plex_playlist"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
|
||||
backend:
|
||||
image: ${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}
|
||||
environment:
|
||||
DATABASE_URL: ${DB_URL:-postgresql://plex_user:plex_password@database:5432/plex_playlist}
|
||||
ENVIRONMENT: production
|
||||
depends_on:
|
||||
database:
|
||||
condition: service_healthy
|
||||
@@ -142,14 +142,10 @@ jobs:
|
||||
|
||||
### Responsibility Split
|
||||
|
||||
- `.gitea/workflows/cicd-start.yaml` owns startup routing and trace propagation.
|
||||
- `.gitea/workflows/cicd-source-checks.yaml` owns early source-level
|
||||
format/lint/type gating before any promotion dispatch.
|
||||
- `.gitea/workflows/docker-build-base.yaml` owns base publication and verification.
|
||||
- `.gitea/workflows/docker-build-main.yaml` owns complete-image publication.
|
||||
- `.gitea/workflows/cicd-checks.yaml` and `.gitea/workflows/cicd-tests.yaml`
|
||||
own post-build CI validation and tests.
|
||||
- Main CI never rebuilds the base image locally.
|
||||
- `.gitea/workflows/cicd.yaml` owns the full CI pipeline: base image publish,
|
||||
CICD image publish, source checks, unit tests, runtime image publication,
|
||||
integration tests, and E2E tests.
|
||||
- The older split workflows remain only as deprecated/manual helper paths.
|
||||
|
||||
### Runtime Boundary Enforcement
|
||||
|
||||
@@ -173,7 +169,7 @@ and deployable runtime artifacts before publishing the complete CICD image.
|
||||
|
||||
Workflow location:
|
||||
|
||||
- `.gitea/workflows/docker-build-main.yaml`
|
||||
- `.gitea/workflows/cicd.yaml`
|
||||
- `Verify deployable runtime boundaries`
|
||||
- `Build and verify deployable runtime image purity`
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ Excluded:
|
||||
### Runtime Artifact Definition
|
||||
|
||||
- Container build source: `Dockerfile.backend`.
|
||||
- Build shape: two-stage build with a dependency stage and a minimal runtime stage.
|
||||
- Runtime base image: `python:3.14-slim`.
|
||||
- Runtime process: `uvicorn backend.main:app --app-dir /app/src --host 0.0.0.0 --port 8000`.
|
||||
- Exposed runtime port: `8000`.
|
||||
@@ -79,6 +80,7 @@ The lockfile in `backend/uv.lock` is the dependency source of truth.
|
||||
### Frontend Runtime Artifact Definition
|
||||
|
||||
- Container build source: `Dockerfile.frontend` (target `production`).
|
||||
- Build shape: two-stage build with a dependency/build stage and a minimal nginx runtime stage.
|
||||
- Runtime base image: `nginx:alpine`.
|
||||
- Runtime process: `nginx -g "daemon off;"`.
|
||||
- Exposed runtime port: `80`.
|
||||
@@ -151,14 +153,14 @@ Current enforcement implemented in CI:
|
||||
|
||||
- Dockerfile target boundary checks:
|
||||
- Script: `scripts/check-dockerfile-boundaries.sh`
|
||||
- Workflow: `.gitea/workflows/docker-build-main.yaml`
|
||||
- Workflow: `.gitea/workflows/cicd.yaml`
|
||||
- Deployable runtime image purity checks:
|
||||
- Script: `scripts/verify-deployable-image-purity.sh`
|
||||
- Workflow: `.gitea/workflows/docker-build-main.yaml`
|
||||
- Workflow: `.gitea/workflows/cicd.yaml`
|
||||
- Checks include binary presence and profile-specific package metadata probes
|
||||
to detect CI/development tooling leakage.
|
||||
- Runtime black-box integration checks against deployed backend container:
|
||||
- Workflow: `.gitea/workflows/cicd-tests.yaml` (`integration-tests` job)
|
||||
- Workflow: `.gitea/workflows/cicd.yaml` (`integration-tests` job)
|
||||
- Inputs: deployable backend commit tag reference and immutable digest
|
||||
reference from main build dispatch.
|
||||
- Assertions: digest/tag consistency and live endpoint behavior for `/`,
|
||||
|
||||
@@ -36,7 +36,7 @@ Scope boundary:
|
||||
|
||||
- Contract definition lives in `DEPLOYABLE_RUNTIME_CONTRACT.md`.
|
||||
- CI enforcement now includes Dockerfile boundary checks and deployable image
|
||||
purity checks in `.gitea/workflows/docker-build-main.yaml`.
|
||||
purity checks in `.gitea/workflows/cicd.yaml`.
|
||||
- Broader workflow redesign and deployment wiring remain out of scope for this
|
||||
repository's runtime contract document and belong to follow-up work under
|
||||
epic #66.
|
||||
@@ -60,8 +60,8 @@ Automated checks:
|
||||
and fails if CI/development binaries or package metadata artifacts are
|
||||
present.
|
||||
|
||||
These checks run in `.gitea/workflows/docker-build-main.yaml` before publishing
|
||||
the complete CICD image.
|
||||
These checks run in `.gitea/workflows/cicd.yaml` before publishing the
|
||||
complete CICD image.
|
||||
|
||||
## Quick Start
|
||||
|
||||
|
||||
@@ -175,10 +175,20 @@ If you want a failover strategy, keep the cached image tagged in the local regis
|
||||
Automation scripts for this workflow live in `scripts/gitea-actions/`:
|
||||
|
||||
- `scripts/gitea-actions/repair_runner_mirror.xsh`
|
||||
Repairs the mirror by pulling from GHCR, tagging/pushing to local registry, and verifying tag presence on each runner host.
|
||||
Repairs mirrors by pulling upstream images, tagging/pushing to local registry, and verifying pullability on each runner host.
|
||||
Current mirror targets:
|
||||
- `kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest`
|
||||
- `kankali.darkhelm.lan:3001/darkhelm.org/renovate:41`
|
||||
- `kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy`
|
||||
|
||||
- `scripts/gitea-actions/check_runner_images.xsh`
|
||||
Verifies host-by-host image presence for both upstream (`GHCR`) and mirrored (`MIRROR`) tags.
|
||||
Verifies host-by-host image presence and pullability for `ubuntu:22.04`, upstream act image (`GHCR`), mirrored act image (`MIRROR`), and Renovate image.
|
||||
|
||||
- `scripts/gitea-actions/runner-prechange-capture.xsh`
|
||||
Captures restart counts, OOM flags, memory state, and runner logs across all hosts.
|
||||
|
||||
- `scripts/gitea-actions/diagnose_runner_startup.xsh`
|
||||
Focused startup diagnostics for runner containers and mirror pull behavior.
|
||||
|
||||
Run from repository root (xonsh):
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
|
||||
const runtimeBaseURL = process.env.PLAYWRIGHT_BASE_URL;
|
||||
const useRuntimeServices = Boolean(runtimeBaseURL);
|
||||
const junitOutputFile = process.env.PLAYWRIGHT_JUNIT_OUTPUT_FILE ?? 'playwright-results.xml';
|
||||
const outputDir = process.env.PLAYWRIGHT_OUTPUT_DIR ?? 'playwright-artifacts';
|
||||
|
||||
export default defineConfig({
|
||||
testDir: './tests/e2e',
|
||||
timeout: process.env.CI ? 60 * 1000 : 30 * 1000, // Longer timeout in CI
|
||||
@@ -10,11 +15,10 @@ export default defineConfig({
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
reporter: process.env.CI
|
||||
? [['list'], ['junit', { outputFile: 'playwright-results.xml' }]]
|
||||
: 'html',
|
||||
reporter: process.env.CI ? [['list'], ['junit', { outputFile: junitOutputFile }]] : 'html',
|
||||
outputDir,
|
||||
use: {
|
||||
baseURL: 'http://localhost:5173',
|
||||
baseURL: runtimeBaseURL ?? 'http://localhost:5173',
|
||||
trace: 'on-first-retry',
|
||||
headless: process.env.CI ? true : false,
|
||||
// CI-specific browser optimizations
|
||||
@@ -78,12 +82,16 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
],
|
||||
webServer: {
|
||||
command: 'yarn dev --host 0.0.0.0 --port 5173 --strictPort',
|
||||
url: 'http://localhost:5173',
|
||||
reuseExistingServer: !process.env.CI,
|
||||
timeout: process.env.CI ? 180 * 1000 : 120 * 1000, // Longer startup timeout in CI
|
||||
stderr: 'pipe',
|
||||
stdout: 'pipe',
|
||||
},
|
||||
...(useRuntimeServices
|
||||
? {}
|
||||
: {
|
||||
webServer: {
|
||||
command: 'yarn dev --host 0.0.0.0 --port 5173 --strictPort',
|
||||
url: 'http://localhost:5173',
|
||||
reuseExistingServer: !process.env.CI,
|
||||
timeout: process.env.CI ? 180 * 1000 : 120 * 1000, // Longer startup timeout in CI
|
||||
stderr: 'pipe',
|
||||
stdout: 'pipe',
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -11,7 +11,14 @@
|
||||
"prConcurrentLimit": 5,
|
||||
"branchConcurrentLimit": 10,
|
||||
"prHourlyLimit": 2,
|
||||
|
||||
"enabledManagers": [
|
||||
"docker-compose",
|
||||
"dockerfile",
|
||||
"github-actions",
|
||||
"npm",
|
||||
"pep621",
|
||||
"regex"
|
||||
],
|
||||
"packageRules": [
|
||||
{
|
||||
"description": "Automerge non-major updates for high-confidence packages",
|
||||
@@ -28,9 +35,24 @@
|
||||
},
|
||||
{
|
||||
"description": "Group Python dev tools updates",
|
||||
"matchManagers": ["uv"],
|
||||
"matchManagers": ["pep621"],
|
||||
"matchCategories": ["python"],
|
||||
"matchDepTypes": ["dev-dependencies"],
|
||||
"matchPackageNames": [
|
||||
"ruff",
|
||||
"pyright",
|
||||
"pydoclint",
|
||||
"pytest",
|
||||
"pytest-asyncio",
|
||||
"pytest-cov",
|
||||
"pytest-mock",
|
||||
"pre-commit",
|
||||
"pyyaml",
|
||||
"yamllint",
|
||||
"toml-sort",
|
||||
"language-formatters-pre-commit-hooks",
|
||||
"xdoctest",
|
||||
"poethepoet"
|
||||
],
|
||||
"groupName": "Python dev tools",
|
||||
"schedule": ["before 9am on monday"]
|
||||
},
|
||||
@@ -76,7 +98,6 @@
|
||||
"reviewersFromCodeOwners": true
|
||||
}
|
||||
],
|
||||
|
||||
"customManagers": [
|
||||
{
|
||||
"description": "Update Python version in Dockerfiles",
|
||||
@@ -97,19 +118,15 @@
|
||||
"versioningTemplate": "node"
|
||||
}
|
||||
],
|
||||
|
||||
"osvVulnerabilityAlerts": true,
|
||||
"osvVulnerabilityAlerts": false,
|
||||
"vulnerabilityAlerts": {
|
||||
"enabled": true,
|
||||
"schedule": ["at any time"]
|
||||
},
|
||||
|
||||
"labels": ["dependencies"],
|
||||
"commitMessagePrefix": "chore:",
|
||||
"commitMessageTopic": "{{depName}}",
|
||||
"commitMessageExtra": "to {{newVersion}}",
|
||||
"commitMessageSuffix": "",
|
||||
|
||||
"prTitle": "{{commitMessagePrefix}} {{commitMessageAction}} {{commitMessageTopic}} {{commitMessageExtra}}",
|
||||
"prBodyTemplate": "This PR contains the following updates:\n\n| Package | Change | Age | Adoption | Passing | Confidence |\n|---|---|---|---|---|---|\n{{#each upgrades}}\n|{{depName}}|{{#if displayFrom}}`{{{displayFrom}}}` -> `{{{displayTo}}}`{{else}}`{{{newVersion}}}`{{/if}}|[](https://docs.renovatebot.com/merge-confidence/)|[](https://docs.renovatebot.com/merge-confidence/)|[](https://docs.renovatebot.com/merge-confidence/)|[](https://docs.renovatebot.com/merge-confidence/)|\n{{/each}}\n\n---\n\n### Release Notes\n\n{{#each upgrades}}\n{{#if hasReleaseNotes}}\n<details>\n<summary>{{depName}}</summary>\n\n{{#each releases}}\n#### {{title}}\n\n{{#if body}}\n{{body}}\n{{/if}}\n{{/each}}\n\n</details>\n{{/if}}\n{{/each}}\n\n### Configuration\n\n📅 **Schedule**: {{schedule}}\n\n🚦 **Automerge**: {{#if isAutomerge}}Enabled{{else}}Disabled{{/if}}\n\n♻ **Rebasing**: {{#if isRebasing}}Rebasing{{else}}Not rebasing{{/if}}\n\n👥 **Reviewers**: {{#if reviewers}}{{reviewers}}{{else}}None{{/if}}\n\n---\n\n*This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate).*"
|
||||
}
|
||||
|
||||
@@ -5,26 +5,63 @@ hosts = [
|
||||
"pi-desktop.darkhelm.lan",
|
||||
]
|
||||
|
||||
mirror_image = "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest"
|
||||
ghcr_runner_image = "ghcr.io/catthehacker/ubuntu:act-latest"
|
||||
renovate_image = "ghcr.io/renovatebot/renovate:41"
|
||||
python_image = "python:3.14-slim"
|
||||
node_image = "node:20-bookworm-slim"
|
||||
|
||||
# Check both upstream and mirrored tags on each runner host.
|
||||
remote_script = """
|
||||
r = !(docker image inspect ubuntu:22.04 > /dev/null 2> /dev/null)
|
||||
print("UBUNTU22:present" if r.returncode == 0 else "UBUNTU22:missing")
|
||||
r = !(docker image inspect ghcr.io/catthehacker/ubuntu:act-latest > /dev/null 2> /dev/null)
|
||||
print("GHCR:present" if r.returncode == 0 else "GHCR:missing")
|
||||
r = !(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null)
|
||||
print("MIRROR_LOCAL:present" if r.returncode == 0 else "MIRROR_LOCAL:missing")
|
||||
def local_state(label, image):
|
||||
r = !(docker image inspect @(image) > /dev/null 2> /dev/null)
|
||||
print(f"{label}:present" if r.returncode == 0 else f"{label}:missing")
|
||||
|
||||
local_state("UBUNTU22", "ubuntu:22.04")
|
||||
local_state("PYTHON", "__PYTHON_IMAGE__")
|
||||
local_state("NODE", "__NODE_IMAGE__")
|
||||
local_state("GHCR", "__GHCR_RUNNER_IMAGE__")
|
||||
local_state("MIRROR_LOCAL", "__MIRROR_IMAGE__")
|
||||
local_state("RENOVATE", "__RENOVATE_IMAGE__")
|
||||
|
||||
r = !(docker info 2> /dev/null | grep -qi "kankali.darkhelm.lan:3001")
|
||||
print("REGISTRY_CONFIG:ok" if r.returncode == 0 else "REGISTRY_CONFIG:missing")
|
||||
|
||||
pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-pull.log 2>&1)
|
||||
if pull.returncode == 0:
|
||||
print("MIRROR_REMOTE:pull-ok")
|
||||
else:
|
||||
mismatch = !(grep -qi "http response to https client" /tmp/mirror-pull.log)
|
||||
print("MIRROR_REMOTE:https-mismatch" if mismatch.returncode == 0 else "MIRROR_REMOTE:pull-failed")
|
||||
def remote_pull(label, image, log_path):
|
||||
pull = !(docker pull @(image) > @(log_path) 2>&1)
|
||||
if pull.returncode == 0:
|
||||
print(f"{label}:pull-ok")
|
||||
return
|
||||
mismatch = !(grep -qi "http response to https client" @(log_path))
|
||||
if mismatch.returncode == 0:
|
||||
print(f"{label}:https-mismatch")
|
||||
else:
|
||||
print(f"{label}:pull-failed")
|
||||
|
||||
remote_pull("UBUNTU22_REMOTE", "ubuntu:22.04", "/tmp/ubuntu22-pull.log")
|
||||
remote_pull("PYTHON_REMOTE", "__PYTHON_IMAGE__", "/tmp/python-pull.log")
|
||||
remote_pull("NODE_REMOTE", "__NODE_IMAGE__", "/tmp/node-pull.log")
|
||||
remote_pull("GHCR_REMOTE", "__GHCR_RUNNER_IMAGE__", "/tmp/ghcr-runner-pull.log")
|
||||
remote_pull("MIRROR_REMOTE", "__MIRROR_IMAGE__", "/tmp/mirror-pull.log")
|
||||
remote_pull("RENOVATE_REMOTE", "__RENOVATE_IMAGE__", "/tmp/renovate-pull.log")
|
||||
|
||||
local_state("UBUNTU22_AFTER_PULL", "ubuntu:22.04")
|
||||
local_state("PYTHON_AFTER_PULL", "__PYTHON_IMAGE__")
|
||||
local_state("NODE_AFTER_PULL", "__NODE_IMAGE__")
|
||||
local_state("GHCR_AFTER_PULL", "__GHCR_RUNNER_IMAGE__")
|
||||
local_state("MIRROR_AFTER_PULL", "__MIRROR_IMAGE__")
|
||||
local_state("RENOVATE_AFTER_PULL", "__RENOVATE_IMAGE__")
|
||||
"""
|
||||
|
||||
remote_script = (
|
||||
remote_script
|
||||
.replace("__MIRROR_IMAGE__", mirror_image)
|
||||
.replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image)
|
||||
.replace("__RENOVATE_IMAGE__", renovate_image)
|
||||
.replace("__PYTHON_IMAGE__", python_image)
|
||||
.replace("__NODE_IMAGE__", node_image)
|
||||
)
|
||||
|
||||
for host in hosts:
|
||||
print(f"\n=== {host} ===")
|
||||
ssh @(host) @(remote_script)
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
# source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago"
|
||||
# source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234"
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import datetime, UTC
|
||||
import subprocess
|
||||
|
||||
hosts = [
|
||||
"kankali.darkhelm.lan",
|
||||
@@ -16,53 +17,79 @@ hosts = [
|
||||
since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago"
|
||||
trace = $ARGS[1] if len($ARGS) > 1 else ""
|
||||
|
||||
print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z")
|
||||
print(f"runner-diagnostics-start={datetime.now(UTC).isoformat()}")
|
||||
print(f"since={since}")
|
||||
print(f"trace_filter={trace or 'none'}")
|
||||
|
||||
remote_script = f'''
|
||||
set -eu
|
||||
SINCE="{since}"
|
||||
TRACE="{trace}"
|
||||
remote_script = """
|
||||
import subprocess
|
||||
|
||||
echo "host=$(hostname -f 2>/dev/null || hostname)"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "kernel=$(uname -r 2>/dev/null || echo unknown)"
|
||||
SINCE = __SINCE__
|
||||
TRACE = __TRACE__
|
||||
|
||||
echo "=== system pressure ==="
|
||||
free -h || true
|
||||
df -h || true
|
||||
uptime || true
|
||||
def run(cmd: str):
|
||||
proc = subprocess.run(cmd, shell=True, text=True, capture_output=True)
|
||||
if proc.stdout:
|
||||
print(proc.stdout, end="")
|
||||
if proc.stderr:
|
||||
print(proc.stderr, end="")
|
||||
return proc.returncode
|
||||
|
||||
echo "=== docker runner containers ==="
|
||||
docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match"
|
||||
run('echo "host=$(hostname -f 2>/dev/null || hostname)"')
|
||||
run('echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"')
|
||||
run('echo "kernel=$(uname -r 2>/dev/null || echo unknown)"')
|
||||
|
||||
echo "=== runner container inspect ==="
|
||||
for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
|
||||
echo "--- container=$c ---"
|
||||
docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true
|
||||
docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true
|
||||
done
|
||||
print('=== system pressure ===')
|
||||
run('free -h || true')
|
||||
run('df -h || true')
|
||||
run('uptime || true')
|
||||
|
||||
echo "=== docker daemon recent log ==="
|
||||
if command -v journalctl >/dev/null 2>&1; then
|
||||
journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true
|
||||
else
|
||||
echo "journalctl_unavailable=true"
|
||||
fi
|
||||
print('=== docker runner containers ===')
|
||||
ps_cmd = "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}' | grep -E 'gitea|runner|act'"
|
||||
if run(ps_cmd) != 0:
|
||||
print('no_runner_container_match')
|
||||
|
||||
echo "=== oom and kill signals ==="
|
||||
dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true
|
||||
print('=== runner container inspect ===')
|
||||
names_cmd = "docker ps -a --format '{{.Names}}' | grep -E 'gitea|runner|act' || true"
|
||||
names_proc = subprocess.run(names_cmd, shell=True, text=True, capture_output=True)
|
||||
container_names = [line.strip() for line in (names_proc.stdout or '').splitlines() if line.strip()]
|
||||
|
||||
if [ -n "$TRACE" ]; then
|
||||
echo "=== trace-filtered logs ==="
|
||||
for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
|
||||
echo "--- trace search in container=$c ---"
|
||||
docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true
|
||||
done
|
||||
fi
|
||||
'''
|
||||
for c in container_names:
|
||||
print(f'--- container={c} ---')
|
||||
run(f"docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' {c} || true")
|
||||
run(f'docker logs --since "{SINCE}" --tail 200 {c} 2>&1 || true')
|
||||
|
||||
print('=== docker daemon recent log ===')
|
||||
if run('command -v journalctl >/dev/null 2>&1') == 0:
|
||||
run(f'journalctl -u docker --since "{SINCE}" --no-pager -n 400 2>/dev/null || true')
|
||||
else:
|
||||
print('journalctl_unavailable=true')
|
||||
|
||||
print('=== oom and kill signals ===')
|
||||
run("dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true")
|
||||
|
||||
if TRACE:
|
||||
print('=== trace-filtered logs ===')
|
||||
for c in container_names:
|
||||
print(f'--- trace search in container={c} ---')
|
||||
run(f'docker logs --since "{SINCE}" {c} 2>&1 | grep -F "{TRACE}" | tail -n 80 || true')
|
||||
"""
|
||||
|
||||
remote_script = remote_script.replace("__SINCE__", repr(since)).replace("__TRACE__", repr(trace))
|
||||
|
||||
for host in hosts:
|
||||
print(f"\n=== {host} ===")
|
||||
ssh @(host) @(remote_script)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ssh", host],
|
||||
input=remote_script,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.stdout:
|
||||
print(result.stdout, end="")
|
||||
if result.stderr:
|
||||
print(result.stderr, end="")
|
||||
except Exception as exc:
|
||||
print(f"runner_diagnostics_error host={host} error={exc}")
|
||||
|
||||
@@ -18,6 +18,15 @@ HOSTS = {
|
||||
"pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
||||
}
|
||||
|
||||
WARMUP_IMAGES = [
|
||||
"ubuntu:22.04",
|
||||
"python:3.14-slim",
|
||||
"node:20-bookworm-slim",
|
||||
"ghcr.io/catthehacker/ubuntu:act-latest",
|
||||
"kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest",
|
||||
"ghcr.io/renovatebot/renovate:41",
|
||||
]
|
||||
|
||||
DEFAULT_MEM_FRACTION = 0.75
|
||||
DEFAULT_JOB_MEM_FRACTION = 0.9
|
||||
DEFAULT_HOST_MEM_SHARE = 0.25
|
||||
@@ -258,6 +267,20 @@ def scp(local_path, host, remote_path):
|
||||
return proc.returncode, proc.stdout + proc.stderr
|
||||
|
||||
|
||||
def warm_runner_images(host):
|
||||
pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES])
|
||||
rc, out = ssh(
|
||||
host,
|
||||
(
|
||||
"set -e\n"
|
||||
"echo 'warming_runner_images_start'\n"
|
||||
f"{pulls}\n"
|
||||
"echo 'warming_runner_images_done'\n"
|
||||
),
|
||||
)
|
||||
return rc, out
|
||||
|
||||
|
||||
def service_block_bounds(lines, start_index):
|
||||
service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" "))
|
||||
block_start = start_index + 1
|
||||
@@ -743,6 +766,13 @@ with tempfile.TemporaryDirectory() as tmpdir:
|
||||
continue
|
||||
print(out.strip())
|
||||
|
||||
rc, out = warm_runner_images(host)
|
||||
if rc != 0:
|
||||
print(out.strip())
|
||||
print("Failed to warm runner images on host")
|
||||
continue
|
||||
print(out.strip())
|
||||
|
||||
rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")
|
||||
print("mounts:", out.strip())
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ hosts = {
|
||||
}
|
||||
|
||||
mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest'
|
||||
ghcr_runner_image = 'ghcr.io/catthehacker/ubuntu:act-latest'
|
||||
renovate_image = 'ghcr.io/renovatebot/renovate:41'
|
||||
do_fix = '--fix' in sys.argv[1:]
|
||||
|
||||
remote_diag = f"""
|
||||
@@ -19,6 +21,9 @@ print('[runner_containers]')
|
||||
r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
|
||||
print(str(r.out).strip())
|
||||
|
||||
r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}}')
|
||||
runner_names = [line.strip() for line in str(r.out).splitlines() if line.strip()]
|
||||
|
||||
print('[registry_config]')
|
||||
r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001')
|
||||
print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing')
|
||||
@@ -36,31 +41,47 @@ else:
|
||||
tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log)
|
||||
print(str(tail_out.out).strip())
|
||||
|
||||
print('[recent_logs_runner1]')
|
||||
log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1)
|
||||
log1_lines = [
|
||||
line for line in str(log1.out).splitlines()
|
||||
if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
|
||||
]
|
||||
print('\\n'.join(log1_lines[-60:]))
|
||||
if not runner_names:
|
||||
print('[recent_logs] no-runner-containers-found')
|
||||
|
||||
print('[recent_logs_runner2]')
|
||||
log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1)
|
||||
log2_lines = [
|
||||
line for line in str(log2.out).splitlines()
|
||||
if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
|
||||
]
|
||||
print('\\n'.join(log2_lines[-60:]))
|
||||
for name in runner_names:
|
||||
print(f'[recent_logs_{name}]')
|
||||
log = !(docker logs --tail 60 @(name) 2>&1)
|
||||
log_lines = [
|
||||
line for line in str(log.out).splitlines()
|
||||
if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline', 'no route to host', 'network is unreachable'])
|
||||
]
|
||||
print('\\n'.join(log_lines[-60:]))
|
||||
"""
|
||||
|
||||
remote_fix = """
|
||||
print('[fix] restarting runners')
|
||||
$(docker compose up -d --force-recreate runner1 runner2)
|
||||
r = !(docker compose up -d --force-recreate act_runner_1 act_runner_2 > /tmp/runner-restart.log 2>&1)
|
||||
if r.returncode != 0:
|
||||
r = !(docker compose up -d --force-recreate runner1 runner2 > /tmp/runner-restart.log 2>&1)
|
||||
if r.returncode != 0:
|
||||
tail_out = !(tail -n 60 /tmp/runner-restart.log)
|
||||
print(str(tail_out.out).strip())
|
||||
raise SystemExit(1)
|
||||
$(sleep 3)
|
||||
print('[fix] warming images')
|
||||
$(docker pull ubuntu:22.04)
|
||||
$(docker pull python:3.14-slim)
|
||||
$(docker pull node:20-bookworm-slim)
|
||||
$(docker pull __GHCR_RUNNER_IMAGE__)
|
||||
$(docker pull __MIRROR_IMAGE__)
|
||||
$(docker pull __RENOVATE_IMAGE__)
|
||||
r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
|
||||
print(str(r.out).strip())
|
||||
"""
|
||||
|
||||
remote_fix = (
|
||||
remote_fix
|
||||
.replace("__MIRROR_IMAGE__", mirror_image)
|
||||
.replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image)
|
||||
.replace("__RENOVATE_IMAGE__", renovate_image)
|
||||
)
|
||||
|
||||
for host, path in hosts.items():
|
||||
print(f"\n===== {host} =====")
|
||||
cmd = f"cd {path}\n{remote_diag}"
|
||||
|
||||
@@ -41,6 +41,17 @@ def first_existing_compose(host, root):
|
||||
return None
|
||||
|
||||
|
||||
def list_runner_containers(host):
|
||||
"""Return actual runner container names present on a host."""
|
||||
rc, text = ssh(
|
||||
host,
|
||||
"docker ps -a --format '{{.Names}}' | grep '^gitea-act-runner-' || true",
|
||||
)
|
||||
if rc != 0 or not text:
|
||||
return []
|
||||
return [line.strip() for line in text.splitlines() if line.strip()]
|
||||
|
||||
|
||||
report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
mkdir -p @(report_dir)
|
||||
report_path = os.path.join(report_dir, "discovery-report.txt")
|
||||
@@ -84,31 +95,29 @@ for host, root in HOSTS:
|
||||
_, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'")
|
||||
data["container_sizes"] = size_text
|
||||
|
||||
# Explicit runner names avoid label-filter parsing issues seen previously.
|
||||
rc1, inspect_1 = ssh(
|
||||
host,
|
||||
"docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'",
|
||||
)
|
||||
rc2, inspect_2 = ssh(
|
||||
host,
|
||||
"docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'",
|
||||
)
|
||||
if rc1 != 0:
|
||||
inspect_1 = "RUNNER1_NOT_FOUND"
|
||||
if rc2 != 0:
|
||||
inspect_2 = "RUNNER2_NOT_FOUND"
|
||||
data["runner_inspect"] = (inspect_1 + "\n" + inspect_2).strip()
|
||||
runner_containers = list_runner_containers(host)
|
||||
data["runner_containers"] = runner_containers
|
||||
|
||||
log1_rc, log1 = ssh(host, "docker logs --tail=40 gitea-act-runner-1")
|
||||
log2_rc, log2 = ssh(host, "docker logs --tail=40 gitea-act-runner-2")
|
||||
if log1_rc != 0:
|
||||
log1 = "RUNNER1_LOGS_UNAVAILABLE"
|
||||
if log2_rc != 0:
|
||||
log2 = "RUNNER2_LOGS_UNAVAILABLE"
|
||||
data["runner_logs"] = {
|
||||
"gitea-act-runner-1": log1,
|
||||
"gitea-act-runner-2": log2,
|
||||
}
|
||||
inspect_lines = []
|
||||
runner_logs = {}
|
||||
if not runner_containers:
|
||||
inspect_lines.append("NO_RUNNER_CONTAINERS_FOUND")
|
||||
for container in runner_containers:
|
||||
rc, inspect_text = ssh(
|
||||
host,
|
||||
f"docker inspect {container} --format 'container={container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} image={{{{.Config.Image}}}}'",
|
||||
)
|
||||
if rc != 0:
|
||||
inspect_text = f"{container}_INSPECT_UNAVAILABLE"
|
||||
inspect_lines.append(inspect_text)
|
||||
|
||||
log_rc, log_text = ssh(host, f"docker logs --tail=40 {container}")
|
||||
if log_rc != 0:
|
||||
log_text = f"{container}_LOGS_UNAVAILABLE"
|
||||
runner_logs[container] = log_text
|
||||
|
||||
data["runner_inspect"] = "\n".join(inspect_lines).strip()
|
||||
data["runner_logs"] = runner_logs
|
||||
|
||||
results[host] = data
|
||||
print("Collected")
|
||||
@@ -134,8 +143,10 @@ with open(report_path, "w", encoding="utf-8") as handle:
|
||||
write_section(handle, "7. MEMORY STATUS", data.get("memory_status", ""))
|
||||
write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", ""))
|
||||
write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", ""))
|
||||
write_section(handle, "10. RUNNER LOGS (1)", data.get("runner_logs", {}).get("gitea-act-runner-1", ""))
|
||||
write_section(handle, "11. RUNNER LOGS (2)", data.get("runner_logs", {}).get("gitea-act-runner-2", ""))
|
||||
for index, container in enumerate(data.get("runner_containers", []), start=10):
|
||||
write_section(handle, f"{index}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, ""))
|
||||
if not data.get("runner_containers"):
|
||||
write_section(handle, "10. RUNNER LOGS", "NO_RUNNER_CONTAINERS_FOUND")
|
||||
|
||||
with open(summary_path, "w", encoding="utf-8") as handle:
|
||||
handle.write("# Infrastructure Discovery Summary\n\n")
|
||||
@@ -144,12 +155,7 @@ with open(summary_path, "w", encoding="utf-8") as handle:
|
||||
handle.write("|------|------|---------|-----------------|\n")
|
||||
|
||||
for host, data in results.items():
|
||||
visible = 0
|
||||
ps = data.get("docker_ps", "")
|
||||
if "gitea-act-runner-1" in ps:
|
||||
visible += 1
|
||||
if "gitea-act-runner-2" in ps:
|
||||
visible += 1
|
||||
visible = len(data.get("runner_containers", []))
|
||||
handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n")
|
||||
|
||||
print("\nDiscovery complete")
|
||||
|
||||
104
scripts/gitea-actions/repair_runner_mirror.xsh
Normal file → Executable file
104
scripts/gitea-actions/repair_runner_mirror.xsh
Normal file → Executable file
@@ -1,3 +1,5 @@
|
||||
#!/usr/bin/env xonsh
|
||||
|
||||
hosts = [
|
||||
"kankali.darkhelm.lan",
|
||||
"zhokq.darkhelm.lan",
|
||||
@@ -5,38 +7,92 @@ hosts = [
|
||||
"pi-desktop.darkhelm.lan",
|
||||
]
|
||||
|
||||
# Pull upstream tag, retag/push to local registry mirror, verify mirror tag.
|
||||
remote_code = """
|
||||
$(docker pull ubuntu:22.04)
|
||||
$(docker pull ghcr.io/catthehacker/ubuntu:act-latest)
|
||||
$(docker tag ghcr.io/catthehacker/ubuntu:act-latest kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest)
|
||||
publisher_host = "kankali.darkhelm.lan"
|
||||
|
||||
push = !(docker push kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-push.log 2>&1)
|
||||
if push.returncode == 0:
|
||||
print("MIRROR_PUSH:ok")
|
||||
# Pull upstream tags on the registry host, publish mirror tags there, then verify pullability on every host.
|
||||
remote_code_template = """
|
||||
mirror_pairs = [
|
||||
("ACT_UBUNTU", "ghcr.io/catthehacker/ubuntu:act-latest", "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest"),
|
||||
("RENOVATE", "ghcr.io/renovatebot/renovate:41", "kankali.darkhelm.lan:3001/darkhelm.org/renovate:41"),
|
||||
("PLAYWRIGHT", "mcr.microsoft.com/playwright:v1.56.1-jammy", "kankali.darkhelm.lan:3001/darkhelm.org/playwright-browsers:v1.56.1-jammy"),
|
||||
]
|
||||
|
||||
is_publisher = __IS_PUBLISHER__
|
||||
|
||||
def classify_failure(prefix, log_path):
|
||||
mismatch = !(grep -qi "http response to https client" @(log_path))
|
||||
if mismatch.returncode == 0:
|
||||
print(f"{prefix}:https-mismatch")
|
||||
else:
|
||||
print(f"{prefix}:failed")
|
||||
tail = !(tail -n 20 @(log_path) 2> /dev/null)
|
||||
if tail.returncode == 0 and str(tail.out).strip():
|
||||
print(f"{prefix}_LOG_START")
|
||||
print(str(tail.out).rstrip())
|
||||
print(f"{prefix}_LOG_END")
|
||||
|
||||
def image_present(image):
|
||||
result = !(docker image inspect @(image) > /dev/null 2>&1)
|
||||
return result.returncode == 0
|
||||
|
||||
ubuntu_pull_log = "/tmp/ubuntu22-pull.log"
|
||||
ubuntu_pull = !(docker pull ubuntu:22.04 > @(ubuntu_pull_log) 2>&1)
|
||||
if ubuntu_pull.returncode == 0:
|
||||
print("UBUNTU22_PULL:ok")
|
||||
else:
|
||||
mismatch = !(grep -qi "http response to https client" /tmp/mirror-push.log)
|
||||
print("MIRROR_PUSH:https-mismatch" if mismatch.returncode == 0 else "MIRROR_PUSH:failed")
|
||||
classify_failure("UBUNTU22_PULL", ubuntu_pull_log)
|
||||
|
||||
# Validate that the mirror is actually pullable from this host's Docker configuration.
|
||||
$(docker image rm kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null || true)
|
||||
pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-verify.log 2>&1)
|
||||
if pull.returncode == 0:
|
||||
print("MIRROR_PULL:ok")
|
||||
else:
|
||||
mismatch = !(grep -qi "http response to https client" /tmp/mirror-verify.log)
|
||||
print("MIRROR_PULL:https-mismatch" if mismatch.returncode == 0 else "MIRROR_PULL:failed")
|
||||
|
||||
if $(docker image inspect ubuntu:22.04):
|
||||
if image_present("ubuntu:22.04"):
|
||||
print("UBUNTU22_PRESENT")
|
||||
else:
|
||||
print("UBUNTU22_MISSING")
|
||||
if $(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest):
|
||||
print("MIRROR_PRESENT")
|
||||
else:
|
||||
print("MIRROR_MISSING")
|
||||
|
||||
for label, source_image, mirror_image in mirror_pairs:
|
||||
print(f"{label}_SOURCE={source_image}")
|
||||
print(f"{label}_MIRROR={mirror_image}")
|
||||
|
||||
src_pull_log = f"/tmp/{label.lower()}-source-pull.log"
|
||||
mirror_push_log = f"/tmp/{label.lower()}-mirror-push.log"
|
||||
mirror_verify_log = f"/tmp/{label.lower()}-mirror-verify.log"
|
||||
|
||||
if is_publisher:
|
||||
src_pull = !(docker pull @(source_image) > @(src_pull_log) 2>&1)
|
||||
if src_pull.returncode == 0:
|
||||
print(f"{label}_SOURCE_PULL:ok")
|
||||
else:
|
||||
classify_failure(f"{label}_SOURCE_PULL", src_pull_log)
|
||||
continue
|
||||
|
||||
tag = !(docker tag @(source_image) @(mirror_image) > /dev/null 2>&1)
|
||||
if tag.returncode != 0:
|
||||
print(f"{label}_TAG:failed")
|
||||
continue
|
||||
|
||||
push = !(docker push @(mirror_image) > @(mirror_push_log) 2>&1)
|
||||
if push.returncode == 0:
|
||||
print(f"{label}_MIRROR_PUSH:ok")
|
||||
else:
|
||||
classify_failure(f"{label}_MIRROR_PUSH", mirror_push_log)
|
||||
continue
|
||||
else:
|
||||
print(f"{label}_SOURCE_PULL:skipped")
|
||||
print(f"{label}_MIRROR_PUSH:skipped")
|
||||
|
||||
# Validate mirror pullability with this host's Docker registry config.
|
||||
rm_image = !(docker image rm @(mirror_image) > /dev/null 2>&1)
|
||||
verify = !(docker pull @(mirror_image) > @(mirror_verify_log) 2>&1)
|
||||
if verify.returncode == 0:
|
||||
print(f"{label}_MIRROR_PULL:ok")
|
||||
else:
|
||||
classify_failure(f"{label}_MIRROR_PULL", mirror_verify_log)
|
||||
|
||||
if image_present(mirror_image):
|
||||
print(f"{label}_MIRROR_PRESENT")
|
||||
else:
|
||||
print(f"{label}_MIRROR_MISSING")
|
||||
"""
|
||||
|
||||
for host in hosts:
|
||||
print(f"\n=== {host} ===")
|
||||
remote_code = remote_code_template.replace("__IS_PUBLISHER__", "True" if host == publisher_host else "False")
|
||||
ssh @(host) @(remote_code)
|
||||
|
||||
@@ -13,6 +13,15 @@ HOST_PATHS = {
|
||||
"pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
||||
}
|
||||
|
||||
WARMUP_IMAGES = [
|
||||
"ubuntu:22.04",
|
||||
"python:3.14-slim",
|
||||
"node:20-bookworm-slim",
|
||||
"ghcr.io/catthehacker/ubuntu:act-latest",
|
||||
"kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest",
|
||||
"ghcr.io/renovatebot/renovate:41",
|
||||
]
|
||||
|
||||
|
||||
def usage(exit_code=1):
|
||||
print("Usage: xonsh rollout-one-runner.xsh <host> [--batch]")
|
||||
@@ -99,5 +108,14 @@ for verify_cmd in [
|
||||
rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'")
|
||||
print(out if rc == 0 else "runner2-removed")
|
||||
|
||||
print("Phase 5: warm images")
|
||||
pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES])
|
||||
rc, out = ssh(f"set -e\n{pulls}")
|
||||
if rc != 0:
|
||||
print("Warmup failed")
|
||||
print(out)
|
||||
raise SystemExit(4)
|
||||
print(out)
|
||||
|
||||
print("One-runner rollout complete for host")
|
||||
print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}")
|
||||
|
||||
@@ -37,6 +37,16 @@ def section(title, body):
|
||||
return f"\n{sep}\n{title}\n{sep}\n{body}\n"
|
||||
|
||||
|
||||
def list_runner_containers(host):
|
||||
rc, text = ssh(
|
||||
host,
|
||||
"docker ps -a --filter name=gitea-act-runner --format '{{.Names}}'",
|
||||
)
|
||||
if rc != 0 or not text:
|
||||
return []
|
||||
return [line.strip() for line in text.splitlines() if line.strip()]
|
||||
|
||||
|
||||
report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
mkdir -p @(report_dir)
|
||||
report_path = os.path.join(report_dir, "capture.txt")
|
||||
@@ -57,39 +67,45 @@ for host, root in HOSTS.items():
|
||||
rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'")
|
||||
data["ps"] = ps_text
|
||||
|
||||
rc1, inspect1 = ssh(
|
||||
host,
|
||||
"docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
|
||||
)
|
||||
rc2, inspect2 = ssh(
|
||||
host,
|
||||
"docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
|
||||
)
|
||||
if rc1 != 0:
|
||||
inspect1 = "runner1-missing"
|
||||
if rc2 != 0:
|
||||
inspect2 = "runner2-missing"
|
||||
data["inspect"] = inspect1 + "\n" + inspect2
|
||||
runner_containers = list_runner_containers(host)
|
||||
data["runner_containers"] = runner_containers
|
||||
|
||||
rc, oom_scan = ssh(
|
||||
host,
|
||||
"docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'",
|
||||
)
|
||||
if rc != 0:
|
||||
oom_scan = "oom-scan-unavailable"
|
||||
data["oom_scan"] = oom_scan
|
||||
inspect_lines = []
|
||||
oom_lines = []
|
||||
if not runner_containers:
|
||||
inspect_lines.append("no-runner-containers-found")
|
||||
oom_lines.append("no-runner-containers-found")
|
||||
|
||||
for container in runner_containers:
|
||||
rc, inspect_text = ssh(
|
||||
host,
|
||||
f"docker inspect {container} --format '{container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}'",
|
||||
)
|
||||
if rc != 0:
|
||||
inspect_text = f"{container}-inspect-unavailable"
|
||||
inspect_lines.append(inspect_text)
|
||||
|
||||
rc, oom_text = ssh(
|
||||
host,
|
||||
f"docker inspect {container} --format '{{{{.Name}}}} oom={{{{.State.OOMKilled}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}'",
|
||||
)
|
||||
if rc != 0:
|
||||
oom_text = f"{container}-oom-scan-unavailable"
|
||||
oom_lines.append(oom_text)
|
||||
|
||||
data["inspect"] = "\n".join(inspect_lines)
|
||||
data["oom_scan"] = "\n".join(oom_lines)
|
||||
|
||||
rc, mem = ssh(host, "free -h")
|
||||
data["mem"] = mem
|
||||
|
||||
rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1")
|
||||
rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2")
|
||||
if rc1 != 0:
|
||||
log1 = "runner1-logs-unavailable"
|
||||
if rc2 != 0:
|
||||
log2 = "runner2-logs-unavailable"
|
||||
data["log1"] = log1
|
||||
data["log2"] = log2
|
||||
runner_logs = {}
|
||||
for container in runner_containers:
|
||||
rc, log_text = ssh(host, f"docker logs --tail={TAIL_LINES} {container}")
|
||||
if rc != 0:
|
||||
log_text = f"{container}-logs-unavailable"
|
||||
runner_logs[container] = log_text
|
||||
data["runner_logs"] = runner_logs
|
||||
|
||||
entries[host] = data
|
||||
|
||||
@@ -108,8 +124,12 @@ with open(report_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", "")))
|
||||
handle.write(section("4. OOM SCAN", data.get("oom_scan", "")))
|
||||
handle.write(section("5. MEMORY", data.get("mem", "")))
|
||||
handle.write(section("6. LOGS RUNNER1", data.get("log1", "")))
|
||||
handle.write(section("7. LOGS RUNNER2", data.get("log2", "")))
|
||||
containers = data.get("runner_containers", [])
|
||||
if not containers:
|
||||
handle.write(section("6. RUNNER LOGS", "no-runner-containers-found"))
|
||||
else:
|
||||
for idx, container in enumerate(containers, start=6):
|
||||
handle.write(section(f"{idx}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, "")))
|
||||
|
||||
print("Capture complete")
|
||||
print(f"Report: {report_path}")
|
||||
|
||||
38
scripts/run-ci-e2e-compose.sh
Normal file
38
scripts/run-ci-e2e-compose.sh
Normal file
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
COMPOSE_FILE="${ROOT_DIR}/compose.ci.e2e.yml"
|
||||
PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-e2e-local}"
|
||||
|
||||
mkdir -p "${ROOT_DIR}/frontend/.playwright-artifacts"
|
||||
|
||||
export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}"
|
||||
export DEPLOYABLE_FRONTEND_IMAGE="${DEPLOYABLE_FRONTEND_IMAGE:-deployable-frontend:local}"
|
||||
export DB_PASSWORD="${DB_PASSWORD:-plex_password}"
|
||||
export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}"
|
||||
export E2E_ARTIFACT_DIR="${E2E_ARTIFACT_DIR:-${ROOT_DIR}/frontend/.playwright-artifacts}"
|
||||
|
||||
cleanup() {
|
||||
docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Running E2E tests against runtime backend/frontend images via compose"
|
||||
echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}"
|
||||
echo "DEPLOYABLE_FRONTEND_IMAGE=${DEPLOYABLE_FRONTEND_IMAGE}"
|
||||
echo "E2E_ARTIFACT_DIR=${E2E_ARTIFACT_DIR}"
|
||||
|
||||
docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend frontend
|
||||
|
||||
docker run --rm \
|
||||
--network "${PROJECT_NAME}_default" \
|
||||
-e CI=true \
|
||||
-e PLAYWRIGHT_BASE_URL="http://frontend:80" \
|
||||
-e PLAYWRIGHT_OUTPUT_DIR=/tmp/playwright-artifacts \
|
||||
-e PLAYWRIGHT_JUNIT_OUTPUT_FILE=/tmp/playwright-artifacts/playwright-results.xml \
|
||||
-v "${ROOT_DIR}/frontend:/workspace/frontend" \
|
||||
-v "${E2E_ARTIFACT_DIR}:/tmp/playwright-artifacts" \
|
||||
mcr.microsoft/playwright:v1.56.1-jammy \
|
||||
bash -lc "cd /workspace/frontend && corepack enable && yarn install --immutable && yarn test:e2e --reporter=list --timeout=90000"
|
||||
96
scripts/run-ci-integration-compose.sh
Normal file
96
scripts/run-ci-integration-compose.sh
Normal file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
COMPOSE_FILE="${ROOT_DIR}/compose.ci.integration.yml"
|
||||
PROJECT_NAME="${COMPOSE_PROJECT_NAME:-plex-int-local}"
|
||||
|
||||
export DEPLOYABLE_BACKEND_IMAGE="${DEPLOYABLE_BACKEND_IMAGE:-deployable-backend:local}"
|
||||
export DB_PASSWORD="${DB_PASSWORD:-plex_password}"
|
||||
export DB_URL="${DB_URL:-postgresql://plex_user:${DB_PASSWORD}@database:5432/plex_playlist}"
|
||||
|
||||
cleanup() {
|
||||
docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v --remove-orphans >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Running runtime integration checks against deployable backend image via compose"
|
||||
echo "DEPLOYABLE_BACKEND_IMAGE=${DEPLOYABLE_BACKEND_IMAGE}"
|
||||
|
||||
docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d database backend
|
||||
|
||||
fetch_backend_endpoint() {
|
||||
endpoint_path="$1"
|
||||
output_file="$2"
|
||||
probe_tmp="$(mktemp)"
|
||||
|
||||
if ! docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" exec -T backend python -c "$(printf '%s\n' \
|
||||
'import sys' \
|
||||
'import urllib.error' \
|
||||
'import urllib.request' \
|
||||
'endpoint_path = sys.argv[1]' \
|
||||
'url = f"http://127.0.0.1:8000{endpoint_path}"' \
|
||||
'try:' \
|
||||
' with urllib.request.urlopen(url, timeout=2) as response:' \
|
||||
' body = response.read().decode("utf-8", errors="replace")' \
|
||||
' print(response.status)' \
|
||||
' print(body)' \
|
||||
'except urllib.error.HTTPError as err:' \
|
||||
' body = err.read().decode("utf-8", errors="replace")' \
|
||||
' print(err.code)' \
|
||||
' print(body)' \
|
||||
'except Exception:' \
|
||||
' print("000")' \
|
||||
' print("")'
|
||||
)" "${endpoint_path}" >"${probe_tmp}" 2>/dev/null
|
||||
then
|
||||
: >"${output_file}"
|
||||
rm -f "${probe_tmp}"
|
||||
echo "000"
|
||||
return 0
|
||||
fi
|
||||
|
||||
http_code="$(head -n 1 "${probe_tmp}")"
|
||||
tail -n +2 "${probe_tmp}" >"${output_file}"
|
||||
rm -f "${probe_tmp}"
|
||||
echo "${http_code}"
|
||||
}
|
||||
|
||||
backend_ready=false
|
||||
for i in $(seq 1 40); do
|
||||
backend_running_count="$(docker compose -p "${PROJECT_NAME}" -f "${COMPOSE_FILE}" ps --status running --services backend | wc -l)"
|
||||
if [ "${backend_running_count}" -eq 0 ]; then
|
||||
echo "❌ Backend container exited before becoming healthy"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
health_code="$(fetch_backend_endpoint "/health" /tmp/blackbox-health.json)"
|
||||
if [ "${health_code}" = "200" ]; then
|
||||
backend_ready=true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "${backend_ready}" != "true" ]; then
|
||||
echo "❌ Backend did not become healthy"
|
||||
cat /tmp/blackbox-health.json 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
root_code="$(fetch_backend_endpoint "/" /tmp/blackbox-root.json)"
|
||||
if [ "${root_code}" != "200" ] || ! grep -q 'Plex Playlist Backend API' /tmp/blackbox-root.json; then
|
||||
echo "❌ Root endpoint validation failed"
|
||||
cat /tmp/blackbox-root.json 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compatibility_code="$(fetch_backend_endpoint "/compatibility" /tmp/blackbox-compatibility.json)"
|
||||
if [ "${compatibility_code}" != "200" ] || ! grep -q '"ok":true' /tmp/blackbox-compatibility.json; then
|
||||
echo "❌ Compatibility endpoint validation failed"
|
||||
cat /tmp/blackbox-compatibility.json 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Runtime integration checks passed"
|
||||
Reference in New Issue
Block a user