fix(ci): avoid base pull false negatives from short timeouts
All checks were successful
CICD Start / Sanity and Base Decision (pull_request) Successful in 59s

This commit is contained in:
copilotcoder
2026-05-29 17:19:21 -04:00
parent c710e39d48
commit f78bd608ce
3 changed files with 42 additions and 28 deletions

View File

@@ -132,17 +132,20 @@ jobs:
echo "Base availability check ${i}/3 for ${BASE_REF_HASH}"
# Manifest inspect is unreliable on some runners; use pull as truth.
if timeout 60 docker pull "${BASE_REF_HASH}" >/tmp/start-base-pull.log 2>&1; then
if timeout 900 docker pull "${BASE_REF_HASH}" >/tmp/start-base-pull.log 2>&1; then
echo "pull_check=ok"
echo "Base rebuild needed: false"
echo "base_needed=false" >> "$GITHUB_OUTPUT"
exit 0
fi
pull_exit=$?
echo "pull_check=failed exit_code=${pull_exit}"
if [ -s /tmp/start-base-pull.log ]; then
tail -n 20 /tmp/start-base-pull.log || true
else
pull_exit=$?
echo "pull_check=failed exit_code=${pull_exit}"
if [ "${pull_exit}" -eq 124 ]; then
echo "Base pull timed out after 900s"
fi
if [ -s /tmp/start-base-pull.log ]; then
tail -n 20 /tmp/start-base-pull.log || true
fi
fi
if [ "${i}" -lt 3 ]; then
sleep 10

View File

@@ -259,16 +259,19 @@ jobs:
for i in 1 2 3; do
echo "Existing base check ${i}/3 for ${BASE_REF_HASH}..."
# Manifest inspect is unreliable on some runners; use pull as truth.
if timeout 60 docker pull "${BASE_REF_HASH}" >/tmp/base-state-pull.log 2>&1; then
if timeout 900 docker pull "${BASE_REF_HASH}" >/tmp/base-state-pull.log 2>&1; then
echo "✓ Immutable base image already exists and is pullable: ${BASE_REF_HASH}"
echo "needs_build=false" >> $GITHUB_OUTPUT
exit 0
fi
pull_exit=$?
echo "Existing base pull check failed with exit code ${pull_exit}"
if [ -s /tmp/base-state-pull.log ]; then
tail -n 20 /tmp/base-state-pull.log || true
else
pull_exit=$?
echo "Existing base pull check failed with exit code ${pull_exit}"
if [ "${pull_exit}" -eq 124 ]; then
echo "Existing base pull timed out after 900s"
fi
if [ -s /tmp/base-state-pull.log ]; then
tail -n 20 /tmp/base-state-pull.log || true
fi
fi
if [ "${i}" -lt 3 ]; then

View File

@@ -120,30 +120,38 @@ jobs:
BASE_IMAGE="${GITEA_REGISTRY}/darkhelm.org/plex-playlist-cicd-base:${BASE_HASH}"
verify_base_image() {
# Fast path: base image already present locally.
if docker image inspect "${BASE_IMAGE}" >/dev/null 2>&1; then
return 0
fi
# Manifest inspect unreliable on these runners; use pull as truth.
timeout 60 docker pull "${BASE_IMAGE}" >/tmp/base-image-pull.log 2>&1
timeout 1800 docker pull "${BASE_IMAGE}" >/tmp/base-image-pull.log 2>&1
}
max_attempts=12
sleep_seconds=10
max_attempts=2
sleep_seconds=15
for i in $(seq 1 "${max_attempts}"); do
echo "Base availability check ${i}/${max_attempts}..."
if verify_base_image; then
echo "✓ Base image available and hash matched: ${BASE_IMAGE}"
break
fi
else
pull_exit=$?
echo "Base pull attempt ${i} failed with exit code ${pull_exit}"
if [ "${pull_exit}" -eq 124 ]; then
echo "Pull timed out after 1800s while downloading base image"
fi
if [ -s /tmp/base-image-pull.log ]; then
tail -n 40 /tmp/base-image-pull.log || true
fi
pull_exit=$?
echo "Base pull attempt ${i} failed with exit code ${pull_exit}"
if [ -s /tmp/base-image-pull.log ]; then
tail -n 20 /tmp/base-image-pull.log || true
if [ "${i}" -eq "${max_attempts}" ]; then
echo "❌ Required immutable base image is not available or mismatched: ${BASE_IMAGE}"
exit 1
fi
sleep "${sleep_seconds}"
fi
if [ "${i}" -eq "${max_attempts}" ]; then
echo "❌ Required immutable base image is not available or mismatched: ${BASE_IMAGE}"
exit 1
fi
sleep "${sleep_seconds}"
done
echo "✓ Base image ready: ${BASE_IMAGE}"