plex-playlist/scripts/gitea-actions/discover-runners.xsh

#!/usr/bin/env xonsh

"""Collect runner baseline details from all four hosts using xonsh-safe SSH commands."""

import os
import sys
from datetime import datetime


HOSTS = [
    ("kankali.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea"),
    ("zhokq.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
    ("urtzul.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
    ("pi-desktop.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
]

BATCH_MODE = "--batch" in sys.argv[1:]


def ssh(host, cmd):
    """Run one command on remote host and return stdout+stderr as text."""
    if BATCH_MODE:
        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    else:
        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
    return out.returncode, str(out.out).rstrip("\n")


def write_section(handle, title, body):
    handle.write("\n" + ("=" * 80) + "\n")
    handle.write(title + "\n")
    handle.write(("=" * 80) + "\n")
    handle.write((body if body else "") + "\n")


def first_existing_compose(host, root):
    for name in ("compose.yml", "docker-compose.yml", "compose.dev.yml"):
        rc, _ = ssh(host, f"test -f {root}/{name}")
        if rc == 0:
            return name
    return None


report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
mkdir -p @(report_dir)
report_path = os.path.join(report_dir, "discovery-report.txt")
summary_path = os.path.join(report_dir, "summary.md")

print("Runner discovery started")
print(f"Report directory: {report_dir}")
print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}")

results = {}

for host, root in HOSTS:
    print(f"\n--- {host} ---")
    data = {"host": host, "root": root}

    rc, _ = ssh(host, f"test -d {root}")
    data["path_exists"] = "EXISTS" if rc == 0 else "NOT_FOUND"
    if rc != 0:
        results[host] = data
        print("Runner path missing")
        continue

    compose_file = first_existing_compose(host, root)
    data["compose_file"] = compose_file or "NOT_FOUND"
    if compose_file:
        _, text = ssh(host, f"cat {root}/{compose_file}")
        data["compose_content"] = text

    env_rc, env_text = ssh(host, f"cat {root}/.env")
    data["env_content"] = env_text if env_rc == 0 else "FILE_NOT_FOUND"

    _, listing = ssh(host, f"ls -la {root}")
    data["directory_listing"] = listing

    _, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'")
    data["docker_ps"] = ps_text

    _, mem_text = ssh(host, "free -h")
    data["memory_status"] = mem_text

    _, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'")
    data["container_sizes"] = size_text

    # Explicit runner names avoid label-filter parsing issues seen previously.
    rc1, inspect_1 = ssh(
        host,
        "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'",
    )
    rc2, inspect_2 = ssh(
        host,
        "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'",
    )
    if rc1 != 0:
        inspect_1 = "RUNNER1_NOT_FOUND"
    if rc2 != 0:
        inspect_2 = "RUNNER2_NOT_FOUND"
    data["runner_inspect"] = (inspect_1 + "\n" + inspect_2).strip()

    log1_rc, log1 = ssh(host, "docker logs --tail=40 gitea-act-runner-1")
    log2_rc, log2 = ssh(host, "docker logs --tail=40 gitea-act-runner-2")
    if log1_rc != 0:
        log1 = "RUNNER1_LOGS_UNAVAILABLE"
    if log2_rc != 0:
        log2 = "RUNNER2_LOGS_UNAVAILABLE"
    data["runner_logs"] = {
        "gitea-act-runner-1": log1,
        "gitea-act-runner-2": log2,
    }

    results[host] = data
    print("Collected")

with open(report_path, "w", encoding="utf-8") as handle:
    handle.write("GITEA ACT RUNNER INFRASTRUCTURE BASELINE DISCOVERY\n")
    handle.write(f"Timestamp: {datetime.now().isoformat()}\n")
    handle.write(f"Hosts: {len(results)}\n\n")

    for host, data in results.items():
        handle.write("\n" + ("#" * 80) + "\n")
        handle.write(f"# HOST: {host}\n")
        handle.write(f"# Path: {data['root']}\n")
        handle.write(("#" * 80) + "\n")

        write_section(handle, "1. PATH EXISTS", data.get("path_exists", "N/A"))
        write_section(handle, "2. COMPOSE FILE", data.get("compose_file", "NOT_FOUND"))
        if data.get("compose_content"):
            write_section(handle, "3. COMPOSE CONTENT", data.get("compose_content", ""))
        write_section(handle, "4. ENV FILE", data.get("env_content", "FILE_NOT_FOUND"))
        write_section(handle, "5. DIRECTORY LISTING", data.get("directory_listing", ""))
        write_section(handle, "6. RUNNING CONTAINERS", data.get("docker_ps", ""))
        write_section(handle, "7. MEMORY STATUS", data.get("memory_status", ""))
        write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", ""))
        write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", ""))
        write_section(handle, "10. RUNNER LOGS (1)", data.get("runner_logs", {}).get("gitea-act-runner-1", ""))
        write_section(handle, "11. RUNNER LOGS (2)", data.get("runner_logs", {}).get("gitea-act-runner-2", ""))

with open(summary_path, "w", encoding="utf-8") as handle:
    handle.write("# Infrastructure Discovery Summary\n\n")
    handle.write(f"Date: {datetime.now().isoformat()}\n\n")
    handle.write("| Host | Path | Compose | Runners Visible |\n")
    handle.write("|------|------|---------|-----------------|\n")

    for host, data in results.items():
        visible = 0
        ps = data.get("docker_ps", "")
        if "gitea-act-runner-1" in ps:
            visible += 1
        if "gitea-act-runner-2" in ps:
            visible += 1
        handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n")

print("\nDiscovery complete")
print(f"Full report: {report_path}")
print(f"Summary: {summary_path}")