diff --git a/scripts/gitea-actions/collect_runner_diagnostics.xsh b/scripts/gitea-actions/collect_runner_diagnostics.xsh new file mode 100644 index 0000000..3251a44 --- /dev/null +++ b/scripts/gitea-actions/collect_runner_diagnostics.xsh @@ -0,0 +1,68 @@ +# Collect actionable runner diagnostics from all known runner hosts. +# Usage examples: +# source scripts/gitea-actions/collect_runner_diagnostics.xsh +# source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago" +# source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234" + +from datetime import datetime + +hosts = [ + "kankali.darkhelm.lan", + "zhokq.darkhelm.lan", + "urtzul.darkhelm.lan", + "pi-desktop.darkhelm.lan", +] + +since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago" +trace = $ARGS[1] if len($ARGS) > 1 else "" + +print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z") +print(f"since={since}") +print(f"trace_filter={trace or 'none'}") + +remote_script = f''' +set -eu +SINCE="{since}" +TRACE="{trace}" + +echo "host=$(hostname -f 2>/dev/null || hostname)" +echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "kernel=$(uname -r 2>/dev/null || echo unknown)" + +echo "=== system pressure ===" +free -h || true +df -h || true +uptime || true + +echo "=== docker runner containers ===" +docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match" + +echo "=== runner container inspect ===" +for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do + echo "--- container=$c ---" + docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true + docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true +done + +echo "=== docker daemon recent log ===" +if command -v journalctl >/dev/null 2>&1; then + journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true +else + echo "journalctl_unavailable=true" +fi + +echo "=== oom and kill signals ===" +dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true + +if [ -n "$TRACE" ]; then + echo "=== trace-filtered logs ===" + for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do + echo "--- trace search in container=$c ---" + docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true + done +fi +''' + +for host in hosts: + print(f"\n=== {host} ===") + ssh @(host) @(remote_script)