plex-playlist/scripts/gitea-actions/runner-prechange-capture.xsh

#!/usr/bin/env xonsh

"""Capture runner restart/OOM/log baseline from all hosts before rollout changes."""

import os
import sys
from datetime import datetime


HOSTS = {
    "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
    "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
    "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
    "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
}

TAIL_LINES = 40
if len(sys.argv) > 1:
    try:
        TAIL_LINES = int(sys.argv[1])
    except ValueError:
        print("Invalid tail line count, using default 40")

BATCH_MODE = "--batch" in sys.argv[1:]


def ssh(host, cmd):
    if BATCH_MODE:
        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    else:
        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    return out.returncode, str(out.out).rstrip("\n")


def section(title, body):
    sep = "=" * 80
    return f"\n{sep}\n{title}\n{sep}\n{body}\n"


def list_runner_containers(host):
    rc, text = ssh(
        host,
        "docker ps -a --filter name=gitea-act-runner --format '{{.Names}}'",
    )
    if rc != 0 or not text:
        return []
    return [line.strip() for line in text.splitlines() if line.strip()]


report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
mkdir -p @(report_dir)
report_path = os.path.join(report_dir, "capture.txt")

print("Runner pre-change capture started")
print(f"Report directory: {report_dir}")
print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}")

entries = {}

for host, root in HOSTS.items():
    print(f"Collecting: {host}")
    data = {"root": root}

    rc, output = ssh(host, f"test -d {root}")
    data["path"] = "EXISTS" if rc == 0 else "NOT_FOUND"

    rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'")
    data["ps"] = ps_text

    runner_containers = list_runner_containers(host)
    data["runner_containers"] = runner_containers

    inspect_lines = []
    oom_lines = []
    if not runner_containers:
        inspect_lines.append("no-runner-containers-found")
        oom_lines.append("no-runner-containers-found")

    for container in runner_containers:
        rc, inspect_text = ssh(
            host,
            f"docker inspect {container} --format '{container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}'",
        )
        if rc != 0:
            inspect_text = f"{container}-inspect-unavailable"
        inspect_lines.append(inspect_text)

        rc, oom_text = ssh(
            host,
            f"docker inspect {container} --format '{{{{.Name}}}} oom={{{{.State.OOMKilled}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}'",
        )
        if rc != 0:
            oom_text = f"{container}-oom-scan-unavailable"
        oom_lines.append(oom_text)

    data["inspect"] = "\n".join(inspect_lines)
    data["oom_scan"] = "\n".join(oom_lines)

    rc, mem = ssh(host, "free -h")
    data["mem"] = mem

    runner_logs = {}
    for container in runner_containers:
        rc, log_text = ssh(host, f"docker logs --tail={TAIL_LINES} {container}")
        if rc != 0:
            log_text = f"{container}-logs-unavailable"
        runner_logs[container] = log_text
    data["runner_logs"] = runner_logs

    entries[host] = data

with open(report_path, "w", encoding="utf-8") as handle:
    handle.write("RUNNER PRE-CHANGE CAPTURE\n")
    handle.write(f"Timestamp: {datetime.now().isoformat()}\n")
    handle.write(f"Tail lines: {TAIL_LINES}\n")

    for host, data in entries.items():
        handle.write("\n" + ("#" * 80) + "\n")
        handle.write(f"# HOST: {host}\n")
        handle.write(f"# PATH: {data['root']}\n")
        handle.write(("#" * 80) + "\n")
        handle.write(section("1. PATH", data.get("path", "")))
        handle.write(section("2. DOCKER PS", data.get("ps", "")))
        handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", "")))
        handle.write(section("4. OOM SCAN", data.get("oom_scan", "")))
        handle.write(section("5. MEMORY", data.get("mem", "")))
        containers = data.get("runner_containers", [])
        if not containers:
            handle.write(section("6. RUNNER LOGS", "no-runner-containers-found"))
        else:
            for idx, container in enumerate(containers, start=6):
                handle.write(section(f"{idx}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, "")))

print("Capture complete")
print(f"Report: {report_path}")