DarkHelm.org/plex-playlist
Public Access
4
1
Fork 0
Code Issues 55 Pull Requests 2 Actions 9 Packages Projects 1 Releases Wiki Activity

ix runtime policy pin mismatch and make dependency audits non-blocking #81

Merged
darkhelm merged 18 commits from fix/backend-runtime-policy-tests into main 2026-07-16 07:40:18 -04:00
Conversation 0 Commits 18 Files Changed 8 +1065 -515
8 changed files with 1065 additions and 515 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1001
.gitea/workflows/cicd.yaml
View File

File diff suppressed because it is too large Load Diff

84
.gitea/workflows/renovate.yml
View File

@@ -26,6 +26,8 @@ jobs:
RENOVATE_IMAGE_FALLBACK: ghcr.io/renovatebot/renovate:41
GITEA_REGISTRY_HOST: kankali.darkhelm.lan
GITEA_REGISTRY_IP: 10.18.75.2
PACKAGE_ACCESS_TOKEN: ${{ secrets.PACKAGE_ACCESS_TOKEN }}
REGISTRY_USER: ${{ github.actor }}
run: |
set -euo pipefail
@@ -60,17 +62,78 @@ jobs:
echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
fi
if [ -n "${PACKAGE_ACCESS_TOKEN:-}" ] && [ -n "${REGISTRY_USER:-}" ]; then
echo "${PACKAGE_ACCESS_TOKEN}" | docker login "http://${GITEA_REGISTRY_HOST}:3001" -u "${REGISTRY_USER}" --password-stdin >/dev/null || true
fi
extract_remote_digest_buildx() {
image_ref="$1"
digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{json .Manifest.Digest}}' 2>/dev/null || true)"
digest="${digest//\"/}"
digest="${digest//$'\n'/}"
digest="${digest//$'\r'/}"
digest="${digest// /}"
if [[ "${digest}" == sha256:* ]]; then
printf '%s\n' "${digest}"
return 0
fi
return 1
}
extract_remote_digest_manifest() {
image_ref="$1"
digest="$(docker manifest inspect "${image_ref}" 2>/dev/null | sed -n 's/.*"digest"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1 || true)"
if [[ "${digest}" == sha256:* ]]; then
printf '%s\n' "${digest}"
return 0
fi
return 1
}
remote_digest() {
image_ref="$1"
if [[ "${image_ref}" == *@sha256:* ]]; then
printf '%s\n' "${image_ref##*@}"
return 0
fi
extract_remote_digest_buildx "${image_ref}" || extract_remote_digest_manifest "${image_ref}"
}
local_digest_matches() {
image_ref="$1"
digest="$2"
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${image_ref}" 2>/dev/null | grep -q "@${digest}$"
}
resolve_candidate() {
image_ref="$1"
if docker image inspect "${image_ref}" >/dev/null 2>&1; then
if digest="$(remote_digest "${image_ref}")"; then
if local_digest_matches "${image_ref}" "${digest}"; then
echo "Using cached current image: ${image_ref}" >&2
printf '%s\n' "${image_ref}"
return 0
fi
echo "Cached image is stale for ${image_ref}; refreshing" >&2
else
echo "Remote digest unavailable for ${image_ref}; attempting refresh pull" >&2
fi
fi
if retry_cmd 3 15 docker pull "${image_ref}" >/dev/null; then
printf '%s\n' "${image_ref}"
return 0
fi
return 1
}
pick_renovate_image() {
for candidate in "${RENOVATE_IMAGE_PRIMARY}" "${RENOVATE_IMAGE_FALLBACK}"; do
echo "Trying Renovate image candidate: ${candidate}" >&2
if docker image inspect "${candidate}" >/dev/null 2>&1; then
echo "Using cached Renovate image: ${candidate}" >&2
printf '%s\n' "${candidate}"
return 0
fi
if retry_cmd 3 15 docker pull "${candidate}" >/dev/null; then
if resolve_candidate "${candidate}" >/dev/null; then
printf '%s\n' "${candidate}"
return 0
fi
@@ -94,11 +157,6 @@ jobs:
echo "RENOVATE_IMAGE=${RENOVATE_IMAGE}" >> "$GITHUB_ENV"
- name: Configure Renovate for Gitea
run: |
echo "=== Configuring Renovate for Gitea ==="
echo "✓ Renovate runtime configuration will be passed through environment and CLI flags"
- name: Run Renovate
env:
# Prefer dedicated Renovate token, then fall back to existing CI tokens.
@@ -490,4 +548,4 @@ jobs:
echo "Check the Dependency Dashboard issue in your repository for detailed results:"
echo "https://dogar.darkhelm.org/DarkHelm.org/plex-playlist/issues"
echo ""
echo "Next scheduled run: Next Monday at 8 AM UTC"
echo "Next scheduled run: Next weekday at 8 AM UTC"

76
Dockerfile.cicd-base
View File

@@ -19,43 +19,47 @@ ENV TZ=America/New_York
# Configure timezone
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Install apt-fast with proper GPG handling
RUN apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
for i in 1 2 3; do \
echo "Attempt $i: Updating package lists..." && \
apt-get update && break || \
(echo "Update attempt $i failed, retrying..." && sleep 10); \
done && \
apt-get install -y \
software-properties-common \
gnupg \
ca-certificates \
curl \
wget \
&& for i in 1 2 3; do \
echo "Attempt $i: Adding apt-fast PPA..." && \
add-apt-repository -y ppa:apt-fast/stable && \
apt-get update && \
apt-get install -y apt-fast && \
break || \
(echo "apt-fast installation attempt $i failed, retrying..." && sleep 10); \
done \
&& rm -rf /var/lib/apt/lists/*
# Configure apt-fast to use apt (not apt-get) with optimized settings
RUN echo 'apt-fast apt-fast/maxdownloads string 10' | debconf-set-selections && \
echo 'apt-fast apt-fast/dlflag boolean true' | debconf-set-selections && \
echo 'apt-fast apt-fast/aptmanager string apt' | debconf-set-selections
# Configure apt timeouts and retries
RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \
echo 'Acquire::http::Timeout "60";' >> /etc/apt/apt.conf.d/80retries && \
echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80retries && \
echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80retries
# Install system dependencies using apt-fast
RUN apt-fast update && apt-fast install -y \
# Bootstrap certificates over HTTP, then enforce HTTPS for all remaining package operations.
RUN set -eux; \
apt-get clean; \
rm -rf /var/lib/apt/lists/*; \
find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 \
| xargs -0 sed -i 's|https://ports.ubuntu.com/ubuntu-ports|http://ports.ubuntu.com/ubuntu-ports|g; s|https://archive.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g; s|https://security.ubuntu.com/ubuntu|http://security.ubuntu.com/ubuntu|g'; \
for i in 1 2 3; do \
echo "Attempt $i: Bootstrapping CA certificates..."; \
if apt-get update && apt-get install -y --no-install-recommends ca-certificates; then \
break; \
fi; \
if [ "$i" -lt 3 ]; then \
echo "Bootstrap attempt $i failed, retrying..."; \
sleep 10; \
else \
exit 1; \
fi; \
done; \
update-ca-certificates; \
find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 \
| xargs -0 sed -i 's|http://ports.ubuntu.com/ubuntu-ports|https://ports.ubuntu.com/ubuntu-ports|g; s|http://archive.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|https://security.ubuntu.com/ubuntu|g'; \
rm -rf /var/lib/apt/lists/*; \
for i in 1 2 3; do \
echo "Attempt $i: Updating package lists over HTTPS..."; \
if apt-get update; then \
break; \
fi; \
if [ "$i" -lt 3 ]; then \
echo "HTTPS update attempt $i failed, retrying..."; \
sleep 10; \
else \
exit 1; \
fi; \
done; \
apt-get install -y --no-install-recommends \
git \
curl \
ca-certificates \
@@ -75,12 +79,15 @@ RUN apt-fast update && apt-fast install -y \
libxkbcommon0 \
libasound2 \
tzdata \
gnupg \
wget \
&& rm -rf /var/lib/apt/lists/*
# Install Python 3.14 with retry and fallback mechanisms
RUN for i in 1 2 3; do \
echo "Attempt $i: Adding deadsnakes PPA..." && \
add-apt-repository -y ppa:deadsnakes/ppa && \
find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 | xargs -0 sed -i 's|http://|https://|g' && \
apt-get update && \
break || \
(echo "Attempt $i failed, retrying in 10s..." && sleep 10); \
@@ -88,7 +95,7 @@ RUN for i in 1 2 3; do \
RUN for i in 1 2 3; do \
echo "Attempt $i: Installing Python 3.14..." && \
timeout 300 apt-fast install -y \
timeout 300 apt-get install -y --no-install-recommends \
python3.14 \
python3.14-venv \
python3.14-dev && \
@@ -102,8 +109,9 @@ RUN for i in 1 2 3; do \
echo "Attempt $i: Installing Node.js 24..." && \
curl -fsSL --connect-timeout 30 --max-time 300 \
https://deb.nodesource.com/setup_24.x | bash - && \
apt-fast update && \
timeout 300 apt-fast install -y nodejs && \
find /etc/apt -type f \( -name 'sources.list' -o -name '*.sources' -o -name '*.list' \) -print0 | xargs -0 sed -i 's|http://|https://|g' && \
apt-get update && \
timeout 300 apt-get install -y --no-install-recommends nodejs && \
break || \
(echo "Attempt $i failed, retrying in 15s..." && sleep 15); \
done && \

3
Dockerfile.frontend
View File

@@ -19,8 +19,7 @@ RUN set -eux; \
done; \
return 1; \
}; \
corepack enable; \
retry 5 corepack prepare yarn@4.10.3 --activate; \
retry 5 npm install -g --force @yarnpkg/cli-dist@4.10.3; \
retry 5 yarn install --immutable
FROM deps AS build

4
backend/src/backend/main.py
View File

@@ -21,10 +21,10 @@ from backend.database import (
)
REQUIRED_PACKAGE_PINS: dict[str, str] = {
"fastapi": "0.120.2",
"fastapi": "0.139.0",
"psycopg": "3.2.12",
"sqlalchemy": "2.0.44",
"uvicorn": "0.38.0",
"uvicorn": "0.51.0",
}

4
backend/tests/integration/test_api.py
View File

@@ -95,8 +95,8 @@ class TestAPIIntegration:
assert "current_python" in payload
assert payload["python_policy_valid"] is True
assert "required_packages" in payload
assert payload["required_packages"]["fastapi"] == "0.120.2"
assert payload["required_packages"]["uvicorn"] == "0.38.0"
assert payload["required_packages"]["fastapi"] == "0.139.0"
assert payload["required_packages"]["uvicorn"] == "0.51.0"
assert payload["package_errors"] == {}
monkeypatch.delenv("BACKEND_REQUIRED_PYTHON", raising=False)

95
backend/tests/test_basic.py
View File

@@ -1,12 +1,22 @@
"""Basic tests for the backend application."""
from typing import cast
from importlib import metadata
from typing import Any, cast
from unittest.mock import AsyncMock
import pytest
from fastapi.testclient import TestClient
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from backend.main import app, get_api_session, read_root
from backend.main import (
app,
compatibility_check,
compatibility_status,
get_api_session,
read_root,
validate_runtime_policy,
)
def test_app_creation():
@@ -56,3 +66,84 @@ def test_typeguard_validation():
# This would fail with typeguard active, but we'll just test the happy path
# to ensure coverage of our code without breaking the test
def test_health_check_db_unavailable():
"""Health endpoint should return unavailable when DB probe fails."""
unhealthy_session = cast("AsyncSession", AsyncMock(spec=AsyncSession))
unhealthy_session.execute = AsyncMock(
side_effect=SQLAlchemyError("database unavailable")
)
async def override_get_session():
"""Provide an unhealthy session dependency override for tests."""
yield unhealthy_session
app.dependency_overrides[get_api_session] = override_get_session
try:
with TestClient(app) as client:
response = client.get("/health")
assert response.status_code == 503
assert response.json() == {
"status": "unhealthy",
"database": "disconnected",
}
finally:
app.dependency_overrides.clear()
def test_compatibility_status_success(monkeypatch: pytest.MonkeyPatch):
"""Compatibility status should be healthy when all checks match policy."""
monkeypatch.setenv("BACKEND_REQUIRED_PYTHON", "3.14")
def installed_version(package_name: str) -> str:
versions = {
"fastapi": "0.139.0",
"psycopg": "3.2.12",
"sqlalchemy": "2.0.44",
"uvicorn": "0.51.0",
}
return versions[package_name]
monkeypatch.setattr("backend.main._installed_version", installed_version)
status = cast("dict[str, Any]", compatibility_status())
assert status["ok"] is True
assert status["package_errors"] == {}
def test_compatibility_status_missing_package(monkeypatch: pytest.MonkeyPatch):
"""Compatibility status should record metadata lookup failures."""
def missing_version(_: str) -> str:
raise metadata.PackageNotFoundError("missing")
monkeypatch.setattr("backend.main._installed_version", missing_version)
status = cast("dict[str, Any]", compatibility_status())
package_checks = cast("dict[str, bool]", status["package_checks"])
package_errors = cast("dict[str, str]", status["package_errors"])
assert status["ok"] is False
assert package_checks["fastapi"] is False
assert "fastapi" in package_errors
def test_validate_runtime_policy_raises(monkeypatch: pytest.MonkeyPatch):
"""Runtime policy validation should fail when compatibility is not ok."""
def invalid_status() -> dict[str, object]:
return {"ok": False}
monkeypatch.setattr("backend.main.compatibility_status", invalid_status)
with pytest.raises(RuntimeError):
validate_runtime_policy()
def test_compatibility_check_returns_status() -> None:
"""Compatibility endpoint helper should return policy payload."""
payload = compatibility_check()
assert "ok" in payload

313
scripts/resolve-mirrored-image.sh Executable file
View File

@@ -0,0 +1,313 @@
#!/bin/bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage: resolve-mirrored-image.sh --image <upstream-or-primary-image> [options]
Required:
--image IMAGE Desired image reference to use locally.
Optional:
--mirror-image IMAGE Mirror image reference to check/pull/push.
--upstream-image IMAGE Upstream source image to import into mirror when mirror is missing.
--registry-user USER Registry username for mirror auth.
--registry-password-env VAR Environment variable name containing mirror registry password/token.
--disable-remote-digest-check Skip remote digest freshness check for local images.
--local-only Only verify local presence; do not pull.
--quiet Suppress informational output except final image ref.
Behavior:
1. If IMAGE exists locally and matches remote digest, use it.
2. Otherwise try to pull IMAGE.
3. If IMAGE pull fails and MIRROR_IMAGE is set, resolve via MIRROR_IMAGE.
4. If MIRROR_IMAGE is missing and UPSTREAM_IMAGE is set, pull upstream, push mirror, then use mirror.
5. Prints the resolved local image reference on stdout.
EOF
}
log() {
if [[ "${QUIET}" != "true" ]]; then
echo "$*" >&2
fi
}
require_arg() {
local value="$1"
local name="$2"
if [[ -z "${value}" ]]; then
echo "Missing required argument: ${name}" >&2
usage >&2
exit 1
fi
}
login_if_needed() {
local image_ref="$1"
local registry password_var password
registry="${image_ref%%/*}"
if [[ -z "${registry}" || "${registry}" == "${image_ref}" ]]; then
return 0
fi
if [[ -z "${REGISTRY_USER}" || -z "${REGISTRY_PASSWORD_ENV}" ]]; then
return 0
fi
password_var="${REGISTRY_PASSWORD_ENV}"
password="${!password_var:-}"
if [[ -z "${password}" ]]; then
log "Skipping docker login for ${registry}: env ${password_var} is empty"
return 0
fi
if [[ -n "${LOGGED_IN_REGISTRIES[${registry}]:-}" ]]; then
return 0
fi
log "Logging into ${registry}"
printf '%s' "${password}" | docker login "http://${registry}" -u "${REGISTRY_USER}" --password-stdin >/dev/null
LOGGED_IN_REGISTRIES["${registry}"]=1
}
image_present_locally() {
local image_ref="$1"
docker image inspect "${image_ref}" >/dev/null 2>&1
}
is_digest_pinned_ref() {
local image_ref="$1"
[[ "${image_ref}" == *@sha256:* ]]
}
extract_remote_digest_buildx() {
local image_ref="$1"
local digest
if ! digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{json .Manifest.Digest}}' 2>/dev/null)"; then
return 1
fi
digest="${digest//\"/}"
digest="${digest//$'\n'/}"
digest="${digest//$'\r'/}"
digest="${digest// /}"
if [[ "${digest}" == sha256:* ]]; then
printf '%s\n' "${digest}"
return 0
fi
return 1
}
extract_remote_digest_manifest() {
local image_ref="$1"
local digest
if ! digest="$(docker manifest inspect "${image_ref}" 2>/dev/null | sed -n 's/.*"digest"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)"; then
return 1
fi
if [[ "${digest}" == sha256:* ]]; then
printf '%s\n' "${digest}"
return 0
fi
return 1
}
get_remote_digest() {
local image_ref="$1"
if is_digest_pinned_ref "${image_ref}"; then
printf '%s\n' "${image_ref##*@}"
return 0
fi
if extract_remote_digest_buildx "${image_ref}"; then
return 0
fi
if extract_remote_digest_manifest "${image_ref}"; then
return 0
fi
return 1
}
local_image_matches_digest() {
local image_ref="$1"
local digest="$2"
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${image_ref}" 2>/dev/null | grep -q "@${digest}$"
}
pull_image() {
local image_ref="$1"
login_if_needed "${image_ref}"
docker pull "${image_ref}" >/dev/null
}
resolve_reference() {
local image_ref="$1"
local remote_digest
if image_present_locally "${image_ref}"; then
if [[ "${DISABLE_REMOTE_DIGEST_CHECK}" == "true" ]] || [[ "${LOCAL_ONLY}" == "true" ]] || is_digest_pinned_ref "${image_ref}"; then
log "Using locally cached image ${image_ref}"
return 0
fi
login_if_needed "${image_ref}"
if remote_digest="$(get_remote_digest "${image_ref}")"; then
if local_image_matches_digest "${image_ref}" "${remote_digest}"; then
log "Using local image ${image_ref}; remote digest matches (${remote_digest})"
return 0
fi
log "Local image ${image_ref} is stale; expected remote digest ${remote_digest}"
else
log "Unable to determine remote digest for ${image_ref}; attempting pull refresh"
fi
fi
if [[ "${LOCAL_ONLY}" == "true" ]]; then
return 1
fi
if pull_image "${image_ref}"; then
log "Pulled image ${image_ref}"
return 0
fi
return 1
}
mirror_available() {
local image_ref="$1"
if pull_image "${image_ref}"; then
return 0
fi
return 1
}
promote_upstream_to_mirror() {
require_arg "${UPSTREAM_IMAGE}" "--upstream-image"
require_arg "${MIRROR_IMAGE}" "--mirror-image"
log "Pulling upstream image ${UPSTREAM_IMAGE}"
docker pull "${UPSTREAM_IMAGE}" >/dev/null
login_if_needed "${MIRROR_IMAGE}"
log "Tagging ${UPSTREAM_IMAGE} as ${MIRROR_IMAGE}"
docker tag "${UPSTREAM_IMAGE}" "${MIRROR_IMAGE}"
log "Pushing ${MIRROR_IMAGE}"
docker push "${MIRROR_IMAGE}" >/dev/null
}
IMAGE=""
MIRROR_IMAGE=""
UPSTREAM_IMAGE=""
REGISTRY_USER=""
REGISTRY_PASSWORD_ENV=""
DISABLE_REMOTE_DIGEST_CHECK="false"
LOCAL_ONLY="false"
QUIET="false"
declare -A LOGGED_IN_REGISTRIES=()
while [[ $# -gt 0 ]]; do
case "$1" in
--image)
IMAGE="${2:-}"
shift 2
;;
--mirror-image)
MIRROR_IMAGE="${2:-}"
shift 2
;;
--upstream-image)
UPSTREAM_IMAGE="${2:-}"
shift 2
;;
--registry-user)
REGISTRY_USER="${2:-}"
shift 2
;;
--registry-password-env)
REGISTRY_PASSWORD_ENV="${2:-}"
shift 2
;;
--disable-remote-digest-check)
DISABLE_REMOTE_DIGEST_CHECK="true"
shift
;;
--local-only)
LOCAL_ONLY="true"
shift
;;
--quiet)
QUIET="true"
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 1
;;
esac
done
require_arg "${IMAGE}" "--image"
if resolve_reference "${IMAGE}"; then
printf '%s\n' "${IMAGE}"
exit 0
fi
if [[ "${LOCAL_ONLY}" == "true" ]]; then
echo "Image unavailable via local-only mode: ${IMAGE}" >&2
exit 1
fi
if [[ -n "${MIRROR_IMAGE}" ]]; then
log "Primary image unavailable; checking mirror ${MIRROR_IMAGE}"
if resolve_reference "${MIRROR_IMAGE}"; then
printf '%s\n' "${MIRROR_IMAGE}"
exit 0
fi
if [[ -n "${UPSTREAM_IMAGE}" ]]; then
log "Mirror image ${MIRROR_IMAGE} missing; promoting from upstream ${UPSTREAM_IMAGE}"
promote_upstream_to_mirror
if ! pull_image "${MIRROR_IMAGE}"; then
echo "Failed to pull promoted mirror image: ${MIRROR_IMAGE}" >&2
exit 1
fi
printf '%s\n' "${MIRROR_IMAGE}"
exit 0
fi
fi
if [[ -n "${UPSTREAM_IMAGE}" ]]; then
log "Falling back to upstream image ${UPSTREAM_IMAGE}"
if resolve_reference "${UPSTREAM_IMAGE}"; then
printf '%s\n' "${UPSTREAM_IMAGE}"
exit 0
fi
fi
echo "Failed to resolve image via primary/mirror/upstream path" >&2
echo "primary=${IMAGE}" >&2
if [[ -n "${MIRROR_IMAGE}" ]]; then
echo "mirror=${MIRROR_IMAGE}" >&2
fi
if [[ -n "${UPSTREAM_IMAGE}" ]]; then
echo "upstream=${UPSTREAM_IMAGE}" >&2
fi
exit 1