plex-playlist/scripts/gitea-actions/runner-prechange-capture.xsh

#!/usr/bin/env xonsh

"""Capture runner restart/OOM/log baseline from all hosts before rollout changes."""

import os
import sys
from datetime import datetime


HOSTS = {
    "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
    "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
    "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
    "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
}

TAIL_LINES = 40
if len(sys.argv) > 1:
    try:
        TAIL_LINES = int(sys.argv[1])
    except ValueError:
        print("Invalid tail line count, using default 40")

BATCH_MODE = "--batch" in sys.argv[1:]


def ssh(host, cmd):
    if BATCH_MODE:
        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    else:
        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    return out.returncode, str(out.out).rstrip("\n")


def section(title, body):
    sep = "=" * 80
    return f"\n{sep}\n{title}\n{sep}\n{body}\n"


report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
mkdir -p @(report_dir)
report_path = os.path.join(report_dir, "capture.txt")

print("Runner pre-change capture started")
print(f"Report directory: {report_dir}")
print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}")

entries = {}

for host, root in HOSTS.items():
    print(f"Collecting: {host}")
    data = {"root": root}

    rc, output = ssh(host, f"test -d {root}")
    data["path"] = "EXISTS" if rc == 0 else "NOT_FOUND"

    rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'")
    data["ps"] = ps_text

    rc1, inspect1 = ssh(
        host,
        "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
    )
    rc2, inspect2 = ssh(
        host,
        "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
    )
    if rc1 != 0:
        inspect1 = "runner1-missing"
    if rc2 != 0:
        inspect2 = "runner2-missing"
    data["inspect"] = inspect1 + "\n" + inspect2

    rc, oom_scan = ssh(
        host,
        "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'",
    )
    if rc != 0:
        oom_scan = "oom-scan-unavailable"
    data["oom_scan"] = oom_scan

    rc, mem = ssh(host, "free -h")
    data["mem"] = mem

    rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1")
    rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2")
    if rc1 != 0:
        log1 = "runner1-logs-unavailable"
    if rc2 != 0:
        log2 = "runner2-logs-unavailable"
    data["log1"] = log1
    data["log2"] = log2

    entries[host] = data

with open(report_path, "w", encoding="utf-8") as handle:
    handle.write("RUNNER PRE-CHANGE CAPTURE\n")
    handle.write(f"Timestamp: {datetime.now().isoformat()}\n")
    handle.write(f"Tail lines: {TAIL_LINES}\n")

    for host, data in entries.items():
        handle.write("\n" + ("#" * 80) + "\n")
        handle.write(f"# HOST: {host}\n")
        handle.write(f"# PATH: {data['root']}\n")
        handle.write(("#" * 80) + "\n")
        handle.write(section("1. PATH", data.get("path", "")))
        handle.write(section("2. DOCKER PS", data.get("ps", "")))
        handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", "")))
        handle.write(section("4. OOM SCAN", data.get("oom_scan", "")))
        handle.write(section("5. MEMORY", data.get("mem", "")))
        handle.write(section("6. LOGS RUNNER1", data.get("log1", "")))
        handle.write(section("7. LOGS RUNNER2", data.get("log2", "")))

print("Capture complete")
print(f"Report: {report_path}")