scripts/gitea-actions/collect_runner_diagnostics.xsh

@@ -4,7 +4,8 @@
 #   source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago"
 #   source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234"
 
-from datetime import datetime
+from datetime import datetime, UTC
+import subprocess
 
 hosts = [
     "kankali.darkhelm.lan",
@@ -16,53 +17,79 @@ hosts = [
 since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago"
 trace = $ARGS[1] if len($ARGS) > 1 else ""
 
-print(f"runner-diagnostics-start={datetime.utcnow().isoformat()}Z")
+print(f"runner-diagnostics-start={datetime.now(UTC).isoformat()}")
 print(f"since={since}")
 print(f"trace_filter={trace or 'none'}")
 
-remote_script = f'''
-set -eu
-SINCE="{since}"
-TRACE="{trace}"
+remote_script = """
+import subprocess
 
-echo "host=$(hostname -f 2>/dev/null || hostname)"
-echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-echo "kernel=$(uname -r 2>/dev/null || echo unknown)"
+SINCE = __SINCE__
+TRACE = __TRACE__
 
-echo "=== system pressure ==="
-free -h || true
-df -h || true
-uptime || true
+def run(cmd: str):
+    proc = subprocess.run(cmd, shell=True, text=True, capture_output=True)
+    if proc.stdout:
+        print(proc.stdout, end="")
+    if proc.stderr:
+        print(proc.stderr, end="")
+    return proc.returncode
 
-echo "=== docker runner containers ==="
-docker ps -a --format '{{{{.Names}}}}|{{{{.Image}}}}|{{{{.Status}}}}' | grep -E 'gitea|runner|act' || echo "no_runner_container_match"
+run('echo "host=$(hostname -f 2>/dev/null || hostname)"')
+run('echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"')
+run('echo "kernel=$(uname -r 2>/dev/null || echo unknown)"')
 
-echo "=== runner container inspect ==="
-for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
-  echo "--- container=$c ---"
-  docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' "$c" || true
-  docker logs --since "$SINCE" --tail 200 "$c" 2>&1 || true
-done
+print('=== system pressure ===')
+run('free -h || true')
+run('df -h || true')
+run('uptime || true')
 
-echo "=== docker daemon recent log ==="
-if command -v journalctl >/dev/null 2>&1; then
-  journalctl -u docker --since "$SINCE" --no-pager -n 400 2>/dev/null || true
-else
-  echo "journalctl_unavailable=true"
-fi
+print('=== docker runner containers ===')
+ps_cmd = "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}' | grep -E 'gitea|runner|act'"
+if run(ps_cmd) != 0:
+    print('no_runner_container_match')
 
-echo "=== oom and kill signals ==="
-dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true
+print('=== runner container inspect ===')
+names_cmd = "docker ps -a --format '{{.Names}}' | grep -E 'gitea|runner|act' || true"
+names_proc = subprocess.run(names_cmd, shell=True, text=True, capture_output=True)
+container_names = [line.strip() for line in (names_proc.stdout or '').splitlines() if line.strip()]
 
-if [ -n "$TRACE" ]; then
-  echo "=== trace-filtered logs ==="
-  for c in $(docker ps -a --format '{{{{.Names}}}}' | grep -E 'gitea|runner|act' || true); do
-    echo "--- trace search in container=$c ---"
-    docker logs --since "$SINCE" "$c" 2>&1 | grep -F "$TRACE" | tail -n 80 || true
-  done
-fi
-'''
+for c in container_names:
+    print(f'--- container={c} ---')
+    run(f"docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' {c} || true")
+    run(f'docker logs --since "{SINCE}" --tail 200 {c} 2>&1 || true')
+
+print('=== docker daemon recent log ===')
+if run('command -v journalctl >/dev/null 2>&1') == 0:
+    run(f'journalctl -u docker --since "{SINCE}" --no-pager -n 400 2>/dev/null || true')
+else:
+    print('journalctl_unavailable=true')
+
+print('=== oom and kill signals ===')
+run("dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true")
+
+if TRACE:
+    print('=== trace-filtered logs ===')
+    for c in container_names:
+        print(f'--- trace search in container={c} ---')
+        run(f'docker logs --since "{SINCE}" {c} 2>&1 | grep -F "{TRACE}" | tail -n 80 || true')
+"""
+
+remote_script = remote_script.replace("__SINCE__", repr(since)).replace("__TRACE__", repr(trace))
 
 for host in hosts:
     print(f"\n=== {host} ===")
-    ssh @(host) @(remote_script)
+    try:
+        result = subprocess.run(
+            ["ssh", host],
+            input=remote_script,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.stdout:
+            print(result.stdout, end="")
+        if result.stderr:
+            print(result.stderr, end="")
+    except Exception as exc:
+        print(f"runner_diagnostics_error host={host} error={exc}")