Trying to get rid of the flakiness for runners (forgot a file).
All checks were successful
CICD Start / Sanity and Base Decision (pull_request) Successful in 13s

Signed-off-by: copilotcoder <copilotcoder@darkhelm.org>
This commit is contained in:
copilotcoder
2026-06-18 09:11:20 -04:00
parent 8353914be3
commit 65480b9160

View File

@@ -0,0 +1,68 @@
# Collect actionable runner diagnostics from all known runner hosts.
# Usage examples:
# source scripts/gitea-actions/collect_runner_diagnostics.xsh
# source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago"
# source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234"
from datetime import datetime
hosts = [
"kankali.darkhelm.lan",
"zhokq.darkhelm.lan",
"urtzul.darkhelm.lan",
"pi-desktop.darkhelm.lan",
]
since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago"
trace = $ARGS[1] if len($ARGS) > 1 else ""
print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z")
print(f"since={since}")
print(f"trace_filter={trace or 'none'}")
remote_script = f'''
set -eu
SINCE="{since}"
TRACE="{trace}"
echo "host=$(hostname -f 2>/dev/null || hostname)"
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "kernel=$(uname -r 2>/dev/null || echo unknown)"
echo "=== system pressure ==="
free -h || true
df -h || true
uptime || true
echo "=== docker runner containers ==="
docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match"
echo "=== runner container inspect ==="
for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
echo "--- container=$c ---"
docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true
docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true
done
echo "=== docker daemon recent log ==="
if command -v journalctl >/dev/null 2>&1; then
journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true
else
echo "journalctl_unavailable=true"
fi
echo "=== oom and kill signals ==="
dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true
if [ -n "$TRACE" ]; then
echo "=== trace-filtered logs ==="
for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
echo "--- trace search in container=$c ---"
docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true
done
fi
'''
for host in hosts:
print(f"\n=== {host} ===")
ssh @(host) @(remote_script)