Trying to get rid of the flakiness for runners (forgot a file).
All checks were successful
CICD Start / Sanity and Base Decision (pull_request) Successful in 13s
All checks were successful
CICD Start / Sanity and Base Decision (pull_request) Successful in 13s
Signed-off-by: copilotcoder <copilotcoder@darkhelm.org>
This commit is contained in:
68
scripts/gitea-actions/collect_runner_diagnostics.xsh
Normal file
68
scripts/gitea-actions/collect_runner_diagnostics.xsh
Normal file
@@ -0,0 +1,68 @@
|
||||
# Collect actionable runner diagnostics from all known runner hosts.
|
||||
# Usage examples:
|
||||
# source scripts/gitea-actions/collect_runner_diagnostics.xsh
|
||||
# source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago"
|
||||
# source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234"
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
hosts = [
|
||||
"kankali.darkhelm.lan",
|
||||
"zhokq.darkhelm.lan",
|
||||
"urtzul.darkhelm.lan",
|
||||
"pi-desktop.darkhelm.lan",
|
||||
]
|
||||
|
||||
since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago"
|
||||
trace = $ARGS[1] if len($ARGS) > 1 else ""
|
||||
|
||||
print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z")
|
||||
print(f"since={since}")
|
||||
print(f"trace_filter={trace or 'none'}")
|
||||
|
||||
remote_script = f'''
|
||||
set -eu
|
||||
SINCE="{since}"
|
||||
TRACE="{trace}"
|
||||
|
||||
echo "host=$(hostname -f 2>/dev/null || hostname)"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "kernel=$(uname -r 2>/dev/null || echo unknown)"
|
||||
|
||||
echo "=== system pressure ==="
|
||||
free -h || true
|
||||
df -h || true
|
||||
uptime || true
|
||||
|
||||
echo "=== docker runner containers ==="
|
||||
docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match"
|
||||
|
||||
echo "=== runner container inspect ==="
|
||||
for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
|
||||
echo "--- container=$c ---"
|
||||
docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true
|
||||
docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true
|
||||
done
|
||||
|
||||
echo "=== docker daemon recent log ==="
|
||||
if command -v journalctl >/dev/null 2>&1; then
|
||||
journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true
|
||||
else
|
||||
echo "journalctl_unavailable=true"
|
||||
fi
|
||||
|
||||
echo "=== oom and kill signals ==="
|
||||
dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true
|
||||
|
||||
if [ -n "$TRACE" ]; then
|
||||
echo "=== trace-filtered logs ==="
|
||||
for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
|
||||
echo "--- trace search in container=$c ---"
|
||||
docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true
|
||||
done
|
||||
fi
|
||||
'''
|
||||
|
||||
for host in hosts:
|
||||
print(f"\n=== {host} ===")
|
||||
ssh @(host) @(remote_script)
|
||||
Reference in New Issue
Block a user