Scripts for restructuring the gitea runners.

Signed-off-by: copilotcoder <copilotcoder@darkhelm.org>

scripts/gitea-actions/deploy-runner-config.xsh

@@ -0,0 +1,132 @@
+#!/usr/bin/env xonsh
+
+"""Deploy shared runner config.yaml to all hosts and wire compose to use CONFIG_FILE."""
+
+import os
+import subprocess
+import tempfile
+from datetime import datetime
+
+TEMPLATE_HOST = "kankali.darkhelm.lan"
+TEMPLATE_PATH = "/home/darkhelm/Projects/DarkHelm.org/gitea/runner-config.template.yaml"
+
+HOSTS = {
+    "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
+    "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+    "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+    "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+}
+
+def ssh(host, cmd):
+    proc = subprocess.run(
+        [
+            "ssh",
+            "-o",
+            "ConnectTimeout=30",
+            "-o",
+            "ServerAliveInterval=30",
+            "-o",
+            "ServerAliveCountMax=6",
+            host,
+            cmd,
+        ],
+        capture_output=True,
+        text=True,
+    )
+    return proc.returncode, proc.stdout + proc.stderr
+
+
+def scp(local_path, host, remote_path):
+    proc = subprocess.run(
+        [
+            "scp",
+            "-o",
+            "ConnectTimeout=30",
+            "-o",
+            "ServerAliveInterval=30",
+            "-o",
+            "ServerAliveCountMax=6",
+            local_path,
+            f"{host}:{remote_path}",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    return proc.returncode, proc.stdout + proc.stderr
+
+
+def build_compose(compose_text):
+    updated = compose_text
+
+    volume_line = "      - ./daemon.json:/etc/docker/daemon.json:ro"
+    add_volume = "      - ./config.yaml:/config.yaml:ro"
+    if add_volume not in updated and volume_line in updated:
+        updated = updated.replace(volume_line, volume_line + "\n" + add_volume)
+
+    force_pull_line = "      - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}"
+    add_cfg_env = "      - CONFIG_FILE=/config.yaml"
+    if add_cfg_env not in updated and force_pull_line in updated:
+        updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env)
+
+    return updated
+
+
+print("Fetching canonical runner config template")
+rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}")
+if rc != 0 or not template.strip():
+    print("Failed to fetch template from kankali")
+    raise SystemExit(2)
+
+now = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+with tempfile.TemporaryDirectory() as tmpdir:
+    local_cfg = os.path.join(tmpdir, "config.yaml")
+    with open(local_cfg, "w", encoding="utf-8") as f:
+        f.write(template)
+
+    for host, root in HOSTS.items():
+        print(f"\n===== {host} =====")
+
+        rc, compose = ssh(host, f"cat {root}/compose.yml")
+        if rc != 0:
+            print("Could not read compose.yml")
+            continue
+
+        new_compose = build_compose(compose)
+
+        rc, _ = ssh(host, f"mkdir -p {root}/backups")
+        rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}")
+        rc, _ = ssh(host, f"test -f {root}/config.yaml")
+        if rc == 0:
+            ssh(host, f"cp -f {root}/config.yaml {root}/backups/config.yaml.{now}")
+
+        local_compose = os.path.join(tmpdir, f"compose.{host}.yml")
+        with open(local_compose, "w", encoding="utf-8") as f:
+            f.write(new_compose)
+
+        rc, out = scp(local_compose, host, f"{root}/compose.yml")
+        if rc != 0:
+            print("Failed to upload compose.yml")
+            print(out)
+            continue
+
+        rc, out = scp(local_cfg, host, f"{root}/config.yaml")
+        if rc != 0:
+            print("Failed to upload config.yaml")
+            print(out)
+            continue
+
+        rc, out = ssh(host, f"grep -n 'config.yaml:/config.yaml:ro' {root}/compose.yml")
+        print(out.strip())
+        rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml")
+        print(out.strip())
+
+        rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1")
+        print(out.strip())
+
+        rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")
+        print("mounts:", out.strip())
+
+        print("Done")
+
+print("\nDeployment complete")

scripts/gitea-actions/diagnose_runner_startup.xsh

@@ -0,0 +1,83 @@
+#!/usr/bin/env xonsh
+
+import sys
+
+hosts = {
+    'kankali.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea',
+    'zhokq.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
+    'urtzul.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
+    'pi-desktop.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
+}
+
+mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest'
+do_fix = '--fix' in sys.argv[1:]
+
+remote_diag = f"""
+print('pwd=' + $(pwd).strip())
+
+print('[runner_containers]')
+r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
+print(str(r.out).strip())
+
+print('[registry_config]')
+r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001')
+print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing')
+
+print('[image_state]')
+r = !(docker image inspect {mirror_image} > /dev/null 2> /dev/null)
+print('MIRROR_LOCAL:present' if r.returncode == 0 else 'MIRROR_LOCAL:missing')
+
+r = !(docker pull {mirror_image} > /tmp/runner-mirror-pull.log 2>&1)
+if r.returncode == 0:
+    print('MIRROR_REMOTE:pull-ok')
+else:
+    mismatch = !(grep -qi 'http response to https client' /tmp/runner-mirror-pull.log)
+    print('MIRROR_REMOTE:https-mismatch' if mismatch.returncode == 0 else 'MIRROR_REMOTE:pull-failed')
+    tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log)
+    print(str(tail_out.out).strip())
+
+print('[recent_logs_runner1]')
+log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1)
+log1_lines = [
+    line for line in str(log1.out).splitlines()
+    if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
+]
+print('\\n'.join(log1_lines[-60:]))
+
+print('[recent_logs_runner2]')
+log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1)
+log2_lines = [
+    line for line in str(log2.out).splitlines()
+    if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
+]
+print('\\n'.join(log2_lines[-60:]))
+"""
+
+remote_fix = """
+print('[fix] restarting runners')
+$(docker compose up -d --force-recreate runner1 runner2)
+$(sleep 3)
+r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
+print(str(r.out).strip())
+"""
+
+for host, path in hosts.items():
+    print(f"\n===== {host} =====")
+    cmd = f"cd {path}\n{remote_diag}"
+    r = !(ssh @(host) @(cmd))
+    if r.returncode != 0:
+        print('DIAG_FAILED')
+        print(str(r.out).strip())
+        continue
+
+    print(str(r.out).strip())
+
+    if do_fix:
+        print('[fix] applying restart')
+        fix_cmd = f"cd {path}\n{remote_fix}"
+        f = !(ssh @(host) @(fix_cmd))
+        if f.returncode != 0:
+            print('FIX_FAILED')
+            print(str(f.out).strip())
+        else:
+            print(str(f.out).strip())

scripts/gitea-actions/discover-runners.xsh

@@ -0,0 +1,157 @@
+#!/usr/bin/env xonsh
+
+"""Collect runner baseline details from all four hosts using xonsh-safe SSH commands."""
+
+import os
+import sys
+from datetime import datetime
+
+
+HOSTS = [
+    ("kankali.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea"),
+    ("zhokq.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
+    ("urtzul.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
+    ("pi-desktop.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"),
+]
+
+BATCH_MODE = "--batch" in sys.argv[1:]
+
+
+def ssh(host, cmd):
+    """Run one command on remote host and return stdout+stderr as text."""
+    if BATCH_MODE:
+        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
+    else:
+        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
+    return out.returncode, str(out.out).rstrip("\n")
+
+
+def write_section(handle, title, body):
+    handle.write("\n" + ("=" * 80) + "\n")
+    handle.write(title + "\n")
+    handle.write(("=" * 80) + "\n")
+    handle.write((body if body else "") + "\n")
+
+
+def first_existing_compose(host, root):
+    for name in ("compose.yml", "docker-compose.yml", "compose.dev.yml"):
+        rc, _ = ssh(host, f"test -f {root}/{name}")
+        if rc == 0:
+            return name
+    return None
+
+
+report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+mkdir -p @(report_dir)
+report_path = os.path.join(report_dir, "discovery-report.txt")
+summary_path = os.path.join(report_dir, "summary.md")
+
+print("Runner discovery started")
+print(f"Report directory: {report_dir}")
+print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}")
+
+results = {}
+
+for host, root in HOSTS:
+    print(f"\n--- {host} ---")
+    data = {"host": host, "root": root}
+
+    rc, _ = ssh(host, f"test -d {root}")
+    data["path_exists"] = "EXISTS" if rc == 0 else "NOT_FOUND"
+    if rc != 0:
+        results[host] = data
+        print("Runner path missing")
+        continue
+
+    compose_file = first_existing_compose(host, root)
+    data["compose_file"] = compose_file or "NOT_FOUND"
+    if compose_file:
+        _, text = ssh(host, f"cat {root}/{compose_file}")
+        data["compose_content"] = text
+
+    env_rc, env_text = ssh(host, f"cat {root}/.env")
+    data["env_content"] = env_text if env_rc == 0 else "FILE_NOT_FOUND"
+
+    _, listing = ssh(host, f"ls -la {root}")
+    data["directory_listing"] = listing
+
+    _, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'")
+    data["docker_ps"] = ps_text
+
+    _, mem_text = ssh(host, "free -h")
+    data["memory_status"] = mem_text
+
+    _, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'")
+    data["container_sizes"] = size_text
+
+    # Explicit runner names avoid label-filter parsing issues seen previously.
+    rc1, inspect_1 = ssh(
+        host,
+        "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'",
+    )
+    rc2, inspect_2 = ssh(
+        host,
+        "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'",
+    )
+    if rc1 != 0:
+        inspect_1 = "RUNNER1_NOT_FOUND"
+    if rc2 != 0:
+        inspect_2 = "RUNNER2_NOT_FOUND"
+    data["runner_inspect"] = (inspect_1 + "\n" + inspect_2).strip()
+
+    log1_rc, log1 = ssh(host, "docker logs --tail=40 gitea-act-runner-1")
+    log2_rc, log2 = ssh(host, "docker logs --tail=40 gitea-act-runner-2")
+    if log1_rc != 0:
+        log1 = "RUNNER1_LOGS_UNAVAILABLE"
+    if log2_rc != 0:
+        log2 = "RUNNER2_LOGS_UNAVAILABLE"
+    data["runner_logs"] = {
+        "gitea-act-runner-1": log1,
+        "gitea-act-runner-2": log2,
+    }
+
+    results[host] = data
+    print("Collected")
+
+with open(report_path, "w", encoding="utf-8") as handle:
+    handle.write("GITEA ACT RUNNER INFRASTRUCTURE BASELINE DISCOVERY\n")
+    handle.write(f"Timestamp: {datetime.now().isoformat()}\n")
+    handle.write(f"Hosts: {len(results)}\n\n")
+
+    for host, data in results.items():
+        handle.write("\n" + ("#" * 80) + "\n")
+        handle.write(f"# HOST: {host}\n")
+        handle.write(f"# Path: {data['root']}\n")
+        handle.write(("#" * 80) + "\n")
+
+        write_section(handle, "1. PATH EXISTS", data.get("path_exists", "N/A"))
+        write_section(handle, "2. COMPOSE FILE", data.get("compose_file", "NOT_FOUND"))
+        if data.get("compose_content"):
+            write_section(handle, "3. COMPOSE CONTENT", data.get("compose_content", ""))
+        write_section(handle, "4. ENV FILE", data.get("env_content", "FILE_NOT_FOUND"))
+        write_section(handle, "5. DIRECTORY LISTING", data.get("directory_listing", ""))
+        write_section(handle, "6. RUNNING CONTAINERS", data.get("docker_ps", ""))
+        write_section(handle, "7. MEMORY STATUS", data.get("memory_status", ""))
+        write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", ""))
+        write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", ""))
+        write_section(handle, "10. RUNNER LOGS (1)", data.get("runner_logs", {}).get("gitea-act-runner-1", ""))
+        write_section(handle, "11. RUNNER LOGS (2)", data.get("runner_logs", {}).get("gitea-act-runner-2", ""))
+
+with open(summary_path, "w", encoding="utf-8") as handle:
+    handle.write("# Infrastructure Discovery Summary\n\n")
+    handle.write(f"Date: {datetime.now().isoformat()}\n\n")
+    handle.write("| Host | Path | Compose | Runners Visible |\n")
+    handle.write("|------|------|---------|-----------------|\n")
+
+    for host, data in results.items():
+        visible = 0
+        ps = data.get("docker_ps", "")
+        if "gitea-act-runner-1" in ps:
+            visible += 1
+        if "gitea-act-runner-2" in ps:
+            visible += 1
+        handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n")
+
+print("\nDiscovery complete")
+print(f"Full report: {report_path}")
+print(f"Summary: {summary_path}")

scripts/gitea-actions/rollout-one-runner.xsh

@@ -0,0 +1,103 @@
+#!/usr/bin/env xonsh
+
+"""Roll out one-runner stabilization on a target host (runner1 active, runner2 removed)."""
+
+import sys
+from datetime import datetime
+
+
+HOST_PATHS = {
+    "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
+    "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+    "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+    "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+}
+
+
+def usage(exit_code=1):
+    print("Usage: xonsh rollout-one-runner.xsh <host> [--batch]")
+    print("Hosts:")
+    for host in HOST_PATHS:
+        print(f"  - {host}")
+    raise SystemExit(exit_code)
+
+
+if len(sys.argv) < 2:
+    usage()
+
+host = sys.argv[1]
+if host not in HOST_PATHS:
+    print(f"Unknown host: {host}")
+    usage()
+
+batch_mode = "--batch" in sys.argv[2:]
+root = HOST_PATHS[host]
+ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+
+def ssh(cmd):
+    if batch_mode:
+        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
+    else:
+        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
+    return out.returncode, str(out.out).rstrip("\n")
+
+
+print(f"Host: {host}")
+print(f"Path: {root}")
+print(f"SSH mode: {'batch' if batch_mode else 'interactive'}")
+print("Phase 1: pre-change snapshot")
+
+rc, out = ssh(f"docker compose -f {root}/compose.yml config --services")
+print(out)
+
+rc, out = ssh("docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'")
+print(out if rc == 0 else "runner1-missing")
+
+rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'")
+print(out if rc == 0 else "runner2-missing")
+
+print("Phase 2: backup compose and env")
+for backup_step in [
+    f"mkdir -p {root}/backups",
+    f"cp -f {root}/compose.yml {root}/backups/compose.yml.{ts}",
+    f"cp -f {root}/.env {root}/backups/.env.{ts}",
+]:
+    rc, out = ssh(backup_step)
+    if out:
+        print(out)
+    if rc != 0:
+        print("Backup failed. Aborting.")
+        raise SystemExit(2)
+print("backups-created")
+
+print("Phase 3: switch to one runner")
+for may_fail in [
+    f"docker compose -f {root}/compose.yml stop act_runner_2",
+    f"docker compose -f {root}/compose.yml rm -f act_runner_2",
+]:
+    rc, out = ssh(may_fail)
+    if out:
+        print(out)
+
+rc, out = ssh(f"docker compose -f {root}/compose.yml up -d act_runner_1")
+if out:
+    print(out)
+if rc != 0:
+    print("Runner switch failed.")
+    raise SystemExit(3)
+
+print("Phase 4: verify")
+for verify_cmd in [
+    "docker ps -a --format 'table {{.Names}}\t{{.Status}}' | grep -E 'NAMES|gitea-act-runner'",
+    "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}}'",
+    "docker logs --tail=30 gitea-act-runner-1 | tail -n 30",
+]:
+    rc, out = ssh(verify_cmd)
+    print(out)
+
+rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'")
+print(out if rc == 0 else "runner2-removed")
+
+print("One-runner rollout complete for host")
+print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}")

scripts/gitea-actions/runner-prechange-capture.xsh

@@ -0,0 +1,115 @@
+#!/usr/bin/env xonsh
+
+"""Capture runner restart/OOM/log baseline from all hosts before rollout changes."""
+
+import os
+import sys
+from datetime import datetime
+
+
+HOSTS = {
+    "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
+    "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+    "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+    "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
+}
+
+TAIL_LINES = 40
+if len(sys.argv) > 1:
+    try:
+        TAIL_LINES = int(sys.argv[1])
+    except ValueError:
+        print("Invalid tail line count, using default 40")
+
+BATCH_MODE = "--batch" in sys.argv[1:]
+
+
+def ssh(host, cmd):
+    if BATCH_MODE:
+        out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
+    else:
+        out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
+    return out.returncode, str(out.out).rstrip("\n")
+
+
+def section(title, body):
+    sep = "=" * 80
+    return f"\n{sep}\n{title}\n{sep}\n{body}\n"
+
+
+report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+mkdir -p @(report_dir)
+report_path = os.path.join(report_dir, "capture.txt")
+
+print("Runner pre-change capture started")
+print(f"Report directory: {report_dir}")
+print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}")
+
+entries = {}
+
+for host, root in HOSTS.items():
+    print(f"Collecting: {host}")
+    data = {"root": root}
+
+    rc, output = ssh(host, f"test -d {root}")
+    data["path"] = "EXISTS" if rc == 0 else "NOT_FOUND"
+
+    rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'")
+    data["ps"] = ps_text
+
+    rc1, inspect1 = ssh(
+        host,
+        "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
+    )
+    rc2, inspect2 = ssh(
+        host,
+        "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
+    )
+    if rc1 != 0:
+        inspect1 = "runner1-missing"
+    if rc2 != 0:
+        inspect2 = "runner2-missing"
+    data["inspect"] = inspect1 + "\n" + inspect2
+
+    rc, oom_scan = ssh(
+        host,
+        "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'",
+    )
+    if rc != 0:
+        oom_scan = "oom-scan-unavailable"
+    data["oom_scan"] = oom_scan
+
+    rc, mem = ssh(host, "free -h")
+    data["mem"] = mem
+
+    rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1")
+    rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2")
+    if rc1 != 0:
+        log1 = "runner1-logs-unavailable"
+    if rc2 != 0:
+        log2 = "runner2-logs-unavailable"
+    data["log1"] = log1
+    data["log2"] = log2
+
+    entries[host] = data
+
+with open(report_path, "w", encoding="utf-8") as handle:
+    handle.write("RUNNER PRE-CHANGE CAPTURE\n")
+    handle.write(f"Timestamp: {datetime.now().isoformat()}\n")
+    handle.write(f"Tail lines: {TAIL_LINES}\n")
+
+    for host, data in entries.items():
+        handle.write("\n" + ("#" * 80) + "\n")
+        handle.write(f"# HOST: {host}\n")
+        handle.write(f"# PATH: {data['root']}\n")
+        handle.write(("#" * 80) + "\n")
+        handle.write(section("1. PATH", data.get("path", "")))
+        handle.write(section("2. DOCKER PS", data.get("ps", "")))
+        handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", "")))
+        handle.write(section("4. OOM SCAN", data.get("oom_scan", "")))
+        handle.write(section("5. MEMORY", data.get("mem", "")))
+        handle.write(section("6. LOGS RUNNER1", data.get("log1", "")))
+        handle.write(section("7. LOGS RUNNER2", data.get("log2", "")))
+
+print("Capture complete")
+print(f"Report: {report_path}")