plex-playlist/scripts/gitea-actions/discover-runners.xsh

#!/usr/bin/env xonsh

"""Collect runner baseline details from all four hosts using xonsh-safe SSH commands."""

import os
import sys
from datetime import datetime


HOSTS = [
    ("kankali.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea"),
    ("zhokq.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
    ("urtzul.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
    ("pi-desktop.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
]

BATCH_MODE = "--batch" in sys.argv[1:]


def ssh(host, cmd):
    """Run one command on remote host and return stdout+stderr as text."""
    if BATCH_MODE:
        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    else:
        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    return out.returncode, str(out.out).rstrip("\n")


def write_section(handle, title, body):
    handle.write("\n" + ("=" * 80) + "\n")
    handle.write(title + "\n")
    handle.write(("=" * 80) + "\n")
    handle.write((body if body else "") + "\n")


def first_existing_compose(host, root):
    for name in ("compose.yml", "docker-compose.yml", "compose.dev.yml"):
        rc, _ = ssh(host, f"test -f {root}/{name}")
        if rc == 0:
            return name
    return None


def list_runner_containers(host):
    """Return actual runner container names present on a host."""
    rc, text = ssh(
        host,
        "docker ps -a --format '{{.Names}}' | grep '^gitea-act-runner-' || true",
    )
    if rc != 0 or not text:
        return []
    return [line.strip() for line in text.splitlines() if line.strip()]


report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
mkdir -p @(report_dir)
report_path = os.path.join(report_dir, "discovery-report.txt")
summary_path = os.path.join(report_dir, "summary.md")

print("Runner discovery started")
print(f"Report directory: {report_dir}")
print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}")

results = {}

for host, root in HOSTS:
    print(f"\n--- {host} ---")
    data = {"host": host, "root": root}

    rc, _ = ssh(host, f"test -d {root}")
    data["path_exists"] = "EXISTS" if rc == 0 else "NOT_FOUND"
    if rc != 0:
        results[host] = data
        print("Runner path missing")
        continue

    compose_file = first_existing_compose(host, root)
    data["compose_file"] = compose_file or "NOT_FOUND"
    if compose_file:
        _, text = ssh(host, f"cat {root}/{compose_file}")
        data["compose_content"] = text

    env_rc, env_text = ssh(host, f"cat {root}/.env")
    data["env_content"] = env_text if env_rc == 0 else "FILE_NOT_FOUND"

    _, listing = ssh(host, f"ls -la {root}")
    data["directory_listing"] = listing

    _, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'")
    data["docker_ps"] = ps_text

    _, mem_text = ssh(host, "free -h")
    data["memory_status"] = mem_text

    _, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'")
    data["container_sizes"] = size_text

    runner_containers = list_runner_containers(host)
    data["runner_containers"] = runner_containers

    inspect_lines = []
    runner_logs = {}
    if not runner_containers:
        inspect_lines.append("NO_RUNNER_CONTAINERS_FOUND")
    for container in runner_containers:
        rc, inspect_text = ssh(
            host,
            f"docker inspect {container} --format 'container={container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} image={{{{.Config.Image}}}}'",
        )
        if rc != 0:
            inspect_text = f"{container}_INSPECT_UNAVAILABLE"
        inspect_lines.append(inspect_text)

        log_rc, log_text = ssh(host, f"docker logs --tail=40 {container}")
        if log_rc != 0:
            log_text = f"{container}_LOGS_UNAVAILABLE"
        runner_logs[container] = log_text

    data["runner_inspect"] = "\n".join(inspect_lines).strip()
    data["runner_logs"] = runner_logs

    results[host] = data
    print("Collected")

with open(report_path, "w", encoding="utf-8") as handle:
    handle.write("GITEA ACT RUNNER INFRASTRUCTURE BASELINE DISCOVERY\n")
    handle.write(f"Timestamp: {datetime.now().isoformat()}\n")
    handle.write(f"Hosts: {len(results)}\n\n")

    for host, data in results.items():
        handle.write("\n" + ("#" * 80) + "\n")
        handle.write(f"# HOST: {host}\n")
        handle.write(f"# Path: {data['root']}\n")
        handle.write(("#" * 80) + "\n")

        write_section(handle, "1. PATH EXISTS", data.get("path_exists", "N/A"))
        write_section(handle, "2. COMPOSE FILE", data.get("compose_file", "NOT_FOUND"))
        if data.get("compose_content"):
            write_section(handle, "3. COMPOSE CONTENT", data.get("compose_content", ""))
        write_section(handle, "4. ENV FILE", data.get("env_content", "FILE_NOT_FOUND"))
        write_section(handle, "5. DIRECTORY LISTING", data.get("directory_listing", ""))
        write_section(handle, "6. RUNNING CONTAINERS", data.get("docker_ps", ""))
        write_section(handle, "7. MEMORY STATUS", data.get("memory_status", ""))
        write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", ""))
        write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", ""))
        for index, container in enumerate(data.get("runner_containers", []), start=10):
            write_section(handle, f"{index}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, ""))
        if not data.get("runner_containers"):
            write_section(handle, "10. RUNNER LOGS", "NO_RUNNER_CONTAINERS_FOUND")

with open(summary_path, "w", encoding="utf-8") as handle:
    handle.write("# Infrastructure Discovery Summary\n\n")
    handle.write(f"Date: {datetime.now().isoformat()}\n\n")
    handle.write("| Host | Path | Compose | Runners Visible |\n")
    handle.write("|------|------|---------|-----------------|\n")

    for host, data in results.items():
        visible = len(data.get("runner_containers", []))
        handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n")

print("\nDiscovery complete")
print(f"Full report: {report_path}")
print(f"Summary: {summary_path}")