plex-playlist/scripts/gitea-actions/collect_runner_diagnostics.xsh

# Collect actionable runner diagnostics from all known runner hosts.
# Usage examples:
#   source scripts/gitea-actions/collect_runner_diagnostics.xsh
#   source scripts/gitea-actions/collect_runner_diagnostics.xsh "2 hours ago"
#   source scripts/gitea-actions/collect_runner_diagnostics.xsh "90 minutes ago" "cicd-checks-1234"

from datetime import datetime, UTC
import subprocess

hosts = [
    "kankali.darkhelm.lan",
    "zhokq.darkhelm.lan",
    "urtzul.darkhelm.lan",
    "pi-desktop.darkhelm.lan",
]

since = $ARGS[0] if len($ARGS) > 0 else "2 hours ago"
trace = $ARGS[1] if len($ARGS) > 1 else ""

print(f"runner-diagnostics-start={datetime.now(UTC).isoformat()}")
print(f"since={since}")
print(f"trace_filter={trace or 'none'}")

remote_script = """
import subprocess

SINCE = __SINCE__
TRACE = __TRACE__

def run(cmd: str):
    proc = subprocess.run(cmd, shell=True, text=True, capture_output=True)
    if proc.stdout:
        print(proc.stdout, end="")
    if proc.stderr:
        print(proc.stderr, end="")
    return proc.returncode

run('echo "host=$(hostname -f 2>/dev/null || hostname)"')
run('echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"')
run('echo "kernel=$(uname -r 2>/dev/null || echo unknown)"')

print('=== system pressure ===')
run('free -h || true')
run('df -h || true')
run('uptime || true')

print('=== docker runner containers ===')
ps_cmd = "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}' | grep -E 'gitea|runner|act'"
if run(ps_cmd) != 0:
    print('no_runner_container_match')

print('=== runner container inspect ===')
names_cmd = "docker ps -a --format '{{.Names}}' | grep -E 'gitea|runner|act' || true"
names_proc = subprocess.run(names_cmd, shell=True, text=True, capture_output=True)
container_names = [line.strip() for line in (names_proc.stdout or '').splitlines() if line.strip()]

for c in container_names:
    print(f'--- container={c} ---')
    run(f"docker inspect --format 'name={{{{.Name}}}} restart={{{{.RestartCount}}}} state={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}' {c} || true")
    run(f'docker logs --since "{SINCE}" --tail 200 {c} 2>&1 || true')

print('=== docker daemon recent log ===')
if run('command -v journalctl >/dev/null 2>&1') == 0:
    run(f'journalctl -u docker --since "{SINCE}" --no-pager -n 400 2>/dev/null || true')
else:
    print('journalctl_unavailable=true')

print('=== oom and kill signals ===')
run("dmesg 2>/dev/null | grep -Ei 'killed process|out of memory|oom' | tail -n 80 || true")

if TRACE:
    print('=== trace-filtered logs ===')
    for c in container_names:
        print(f'--- trace search in container={c} ---')
        run(f'docker logs --since "{SINCE}" {c} 2>&1 | grep -F "{TRACE}" | tail -n 80 || true')
"""

remote_script = remote_script.replace("__SINCE__", repr(since)).replace("__TRACE__", repr(trace))

for host in hosts:
    print(f"\n=== {host} ===")
    try:
        result = subprocess.run(
            ["ssh", host],
            input=remote_script,
            capture_output=True,
            text=True,
            check=False,
        )
        if result.stdout:
            print(result.stdout, end="")
        if result.stderr:
            print(result.stderr, end="")
    except Exception as exc:
        print(f"runner_diagnostics_error host={host} error={exc}")