diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh new file mode 100755 index 0000000..9ea40fa --- /dev/null +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -0,0 +1,132 @@ +#!/usr/bin/env xonsh + +"""Deploy shared runner config.yaml to all hosts and wire compose to use CONFIG_FILE.""" + +import os +import subprocess +import tempfile +from datetime import datetime + +TEMPLATE_HOST = "kankali.darkhelm.lan" +TEMPLATE_PATH = "/home/darkhelm/Projects/DarkHelm.org/gitea/runner-config.template.yaml" + +HOSTS = { + "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea", + "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", + "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", + "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", +} + +def ssh(host, cmd): + proc = subprocess.run( + [ + "ssh", + "-o", + "ConnectTimeout=30", + "-o", + "ServerAliveInterval=30", + "-o", + "ServerAliveCountMax=6", + host, + cmd, + ], + capture_output=True, + text=True, + ) + return proc.returncode, proc.stdout + proc.stderr + + +def scp(local_path, host, remote_path): + proc = subprocess.run( + [ + "scp", + "-o", + "ConnectTimeout=30", + "-o", + "ServerAliveInterval=30", + "-o", + "ServerAliveCountMax=6", + local_path, + f"{host}:{remote_path}", + ], + capture_output=True, + text=True, + ) + return proc.returncode, proc.stdout + proc.stderr + + +def build_compose(compose_text): + updated = compose_text + + volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro" + add_volume = " - ./config.yaml:/config.yaml:ro" + if add_volume not in updated and volume_line in updated: + updated = updated.replace(volume_line, volume_line + "\n" + add_volume) + + force_pull_line = " - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}" + add_cfg_env = " - CONFIG_FILE=/config.yaml" + if add_cfg_env not in updated and force_pull_line in updated: + updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env) + + return updated + + +print("Fetching canonical runner config template") +rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}") +if rc != 0 or not template.strip(): + print("Failed to fetch template from kankali") + raise SystemExit(2) + +now = datetime.now().strftime("%Y%m%d_%H%M%S") + +with tempfile.TemporaryDirectory() as tmpdir: + local_cfg = os.path.join(tmpdir, "config.yaml") + with open(local_cfg, "w", encoding="utf-8") as f: + f.write(template) + + for host, root in HOSTS.items(): + print(f"\n===== {host} =====") + + rc, compose = ssh(host, f"cat {root}/compose.yml") + if rc != 0: + print("Could not read compose.yml") + continue + + new_compose = build_compose(compose) + + rc, _ = ssh(host, f"mkdir -p {root}/backups") + rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}") + rc, _ = ssh(host, f"test -f {root}/config.yaml") + if rc == 0: + ssh(host, f"cp -f {root}/config.yaml {root}/backups/config.yaml.{now}") + + local_compose = os.path.join(tmpdir, f"compose.{host}.yml") + with open(local_compose, "w", encoding="utf-8") as f: + f.write(new_compose) + + rc, out = scp(local_compose, host, f"{root}/compose.yml") + if rc != 0: + print("Failed to upload compose.yml") + print(out) + continue + + rc, out = scp(local_cfg, host, f"{root}/config.yaml") + if rc != 0: + print("Failed to upload config.yaml") + print(out) + continue + + rc, out = ssh(host, f"grep -n 'config.yaml:/config.yaml:ro' {root}/compose.yml") + print(out.strip()) + rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml") + print(out.strip()) + + rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1") + print(out.strip()) + + rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'") + print("mounts:", out.strip()) + + print("Done") + +print("\nDeployment complete") diff --git a/scripts/gitea-actions/diagnose_runner_startup.xsh b/scripts/gitea-actions/diagnose_runner_startup.xsh new file mode 100755 index 0000000..8339c98 --- /dev/null +++ b/scripts/gitea-actions/diagnose_runner_startup.xsh @@ -0,0 +1,83 @@ +#!/usr/bin/env xonsh + +import sys + +hosts = { + 'kankali.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea', + 'zhokq.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner', + 'urtzul.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner', + 'pi-desktop.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner', +} + +mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest' +do_fix = '--fix' in sys.argv[1:] + +remote_diag = f""" +print('pwd=' + $(pwd).strip()) + +print('[runner_containers]') +r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') +print(str(r.out).strip()) + +print('[registry_config]') +r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001') +print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing') + +print('[image_state]') +r = !(docker image inspect {mirror_image} > /dev/null 2> /dev/null) +print('MIRROR_LOCAL:present' if r.returncode == 0 else 'MIRROR_LOCAL:missing') + +r = !(docker pull {mirror_image} > /tmp/runner-mirror-pull.log 2>&1) +if r.returncode == 0: + print('MIRROR_REMOTE:pull-ok') +else: + mismatch = !(grep -qi 'http response to https client' /tmp/runner-mirror-pull.log) + print('MIRROR_REMOTE:https-mismatch' if mismatch.returncode == 0 else 'MIRROR_REMOTE:pull-failed') + tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log) + print(str(tail_out.out).strip()) + +print('[recent_logs_runner1]') +log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1) +log1_lines = [ + line for line in str(log1.out).splitlines() + if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) +] +print('\\n'.join(log1_lines[-60:])) + +print('[recent_logs_runner2]') +log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1) +log2_lines = [ + line for line in str(log2.out).splitlines() + if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) +] +print('\\n'.join(log2_lines[-60:])) +""" + +remote_fix = """ +print('[fix] restarting runners') +$(docker compose up -d --force-recreate runner1 runner2) +$(sleep 3) +r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') +print(str(r.out).strip()) +""" + +for host, path in hosts.items(): + print(f"\n===== {host} =====") + cmd = f"cd {path}\n{remote_diag}" + r = !(ssh @(host) @(cmd)) + if r.returncode != 0: + print('DIAG_FAILED') + print(str(r.out).strip()) + continue + + print(str(r.out).strip()) + + if do_fix: + print('[fix] applying restart') + fix_cmd = f"cd {path}\n{remote_fix}" + f = !(ssh @(host) @(fix_cmd)) + if f.returncode != 0: + print('FIX_FAILED') + print(str(f.out).strip()) + else: + print(str(f.out).strip()) diff --git a/scripts/gitea-actions/discover-runners.xsh b/scripts/gitea-actions/discover-runners.xsh new file mode 100755 index 0000000..137963c --- /dev/null +++ b/scripts/gitea-actions/discover-runners.xsh @@ -0,0 +1,157 @@ +#!/usr/bin/env xonsh + +"""Collect runner baseline details from all four hosts using xonsh-safe SSH commands.""" + +import os +import sys +from datetime import datetime + + +HOSTS = [ + ("kankali.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea"), + ("zhokq.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"), + ("urtzul.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"), + ("pi-desktop.darkhelm.lan", "/home/darkhelm/Projects/DarkHelm.org/gitea-runner"), +] + +BATCH_MODE = "--batch" in sys.argv[1:] + + +def ssh(host, cmd): + """Run one command on remote host and return stdout+stderr as text.""" + if BATCH_MODE: + out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) + else: + out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) + return out.returncode, str(out.out).rstrip("\n") + + +def write_section(handle, title, body): + handle.write("\n" + ("=" * 80) + "\n") + handle.write(title + "\n") + handle.write(("=" * 80) + "\n") + handle.write((body if body else "") + "\n") + + +def first_existing_compose(host, root): + for name in ("compose.yml", "docker-compose.yml", "compose.dev.yml"): + rc, _ = ssh(host, f"test -f {root}/{name}") + if rc == 0: + return name + return None + + +report_dir = f"/tmp/runner-discovery-{datetime.now().strftime('%Y%m%d_%H%M%S')}" +mkdir -p @(report_dir) +report_path = os.path.join(report_dir, "discovery-report.txt") +summary_path = os.path.join(report_dir, "summary.md") + +print("Runner discovery started") +print(f"Report directory: {report_dir}") +print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}") + +results = {} + +for host, root in HOSTS: + print(f"\n--- {host} ---") + data = {"host": host, "root": root} + + rc, _ = ssh(host, f"test -d {root}") + data["path_exists"] = "EXISTS" if rc == 0 else "NOT_FOUND" + if rc != 0: + results[host] = data + print("Runner path missing") + continue + + compose_file = first_existing_compose(host, root) + data["compose_file"] = compose_file or "NOT_FOUND" + if compose_file: + _, text = ssh(host, f"cat {root}/{compose_file}") + data["compose_content"] = text + + env_rc, env_text = ssh(host, f"cat {root}/.env") + data["env_content"] = env_text if env_rc == 0 else "FILE_NOT_FOUND" + + _, listing = ssh(host, f"ls -la {root}") + data["directory_listing"] = listing + + _, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'") + data["docker_ps"] = ps_text + + _, mem_text = ssh(host, "free -h") + data["memory_status"] = mem_text + + _, size_text = ssh(host, "docker ps -a --size --format 'table {{.Names}}\t{{.Size}}'") + data["container_sizes"] = size_text + + # Explicit runner names avoid label-filter parsing issues seen previously. + rc1, inspect_1 = ssh( + host, + "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'", + ) + rc2, inspect_2 = ssh( + host, + "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} image={{.Config.Image}}'", + ) + if rc1 != 0: + inspect_1 = "RUNNER1_NOT_FOUND" + if rc2 != 0: + inspect_2 = "RUNNER2_NOT_FOUND" + data["runner_inspect"] = (inspect_1 + "\n" + inspect_2).strip() + + log1_rc, log1 = ssh(host, "docker logs --tail=40 gitea-act-runner-1") + log2_rc, log2 = ssh(host, "docker logs --tail=40 gitea-act-runner-2") + if log1_rc != 0: + log1 = "RUNNER1_LOGS_UNAVAILABLE" + if log2_rc != 0: + log2 = "RUNNER2_LOGS_UNAVAILABLE" + data["runner_logs"] = { + "gitea-act-runner-1": log1, + "gitea-act-runner-2": log2, + } + + results[host] = data + print("Collected") + +with open(report_path, "w", encoding="utf-8") as handle: + handle.write("GITEA ACT RUNNER INFRASTRUCTURE BASELINE DISCOVERY\n") + handle.write(f"Timestamp: {datetime.now().isoformat()}\n") + handle.write(f"Hosts: {len(results)}\n\n") + + for host, data in results.items(): + handle.write("\n" + ("#" * 80) + "\n") + handle.write(f"# HOST: {host}\n") + handle.write(f"# Path: {data['root']}\n") + handle.write(("#" * 80) + "\n") + + write_section(handle, "1. PATH EXISTS", data.get("path_exists", "N/A")) + write_section(handle, "2. COMPOSE FILE", data.get("compose_file", "NOT_FOUND")) + if data.get("compose_content"): + write_section(handle, "3. COMPOSE CONTENT", data.get("compose_content", "")) + write_section(handle, "4. ENV FILE", data.get("env_content", "FILE_NOT_FOUND")) + write_section(handle, "5. DIRECTORY LISTING", data.get("directory_listing", "")) + write_section(handle, "6. RUNNING CONTAINERS", data.get("docker_ps", "")) + write_section(handle, "7. MEMORY STATUS", data.get("memory_status", "")) + write_section(handle, "8. CONTAINER SIZES", data.get("container_sizes", "")) + write_section(handle, "9. RUNNER INSPECT", data.get("runner_inspect", "")) + write_section(handle, "10. RUNNER LOGS (1)", data.get("runner_logs", {}).get("gitea-act-runner-1", "")) + write_section(handle, "11. RUNNER LOGS (2)", data.get("runner_logs", {}).get("gitea-act-runner-2", "")) + +with open(summary_path, "w", encoding="utf-8") as handle: + handle.write("# Infrastructure Discovery Summary\n\n") + handle.write(f"Date: {datetime.now().isoformat()}\n\n") + handle.write("| Host | Path | Compose | Runners Visible |\n") + handle.write("|------|------|---------|-----------------|\n") + + for host, data in results.items(): + visible = 0 + ps = data.get("docker_ps", "") + if "gitea-act-runner-1" in ps: + visible += 1 + if "gitea-act-runner-2" in ps: + visible += 1 + handle.write(f"| {host} | {data.get('path_exists', 'N/A')} | {data.get('compose_file', 'N/A')} | {visible} |\n") + +print("\nDiscovery complete") +print(f"Full report: {report_path}") +print(f"Summary: {summary_path}") diff --git a/scripts/gitea-actions/rollout-one-runner.xsh b/scripts/gitea-actions/rollout-one-runner.xsh new file mode 100755 index 0000000..afb5578 --- /dev/null +++ b/scripts/gitea-actions/rollout-one-runner.xsh @@ -0,0 +1,103 @@ +#!/usr/bin/env xonsh + +"""Roll out one-runner stabilization on a target host (runner1 active, runner2 removed).""" + +import sys +from datetime import datetime + + +HOST_PATHS = { + "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea", + "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", + "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", + "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", +} + + +def usage(exit_code=1): + print("Usage: xonsh rollout-one-runner.xsh [--batch]") + print("Hosts:") + for host in HOST_PATHS: + print(f" - {host}") + raise SystemExit(exit_code) + + +if len(sys.argv) < 2: + usage() + +host = sys.argv[1] +if host not in HOST_PATHS: + print(f"Unknown host: {host}") + usage() + +batch_mode = "--batch" in sys.argv[2:] +root = HOST_PATHS[host] +ts = datetime.now().strftime("%Y%m%d_%H%M%S") + + +def ssh(cmd): + if batch_mode: + out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) + else: + out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) + return out.returncode, str(out.out).rstrip("\n") + + +print(f"Host: {host}") +print(f"Path: {root}") +print(f"SSH mode: {'batch' if batch_mode else 'interactive'}") +print("Phase 1: pre-change snapshot") + +rc, out = ssh(f"docker compose -f {root}/compose.yml config --services") +print(out) + +rc, out = ssh("docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'") +print(out if rc == 0 else "runner1-missing") + +rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'") +print(out if rc == 0 else "runner2-missing") + +print("Phase 2: backup compose and env") +for backup_step in [ + f"mkdir -p {root}/backups", + f"cp -f {root}/compose.yml {root}/backups/compose.yml.{ts}", + f"cp -f {root}/.env {root}/backups/.env.{ts}", +]: + rc, out = ssh(backup_step) + if out: + print(out) + if rc != 0: + print("Backup failed. Aborting.") + raise SystemExit(2) +print("backups-created") + +print("Phase 3: switch to one runner") +for may_fail in [ + f"docker compose -f {root}/compose.yml stop act_runner_2", + f"docker compose -f {root}/compose.yml rm -f act_runner_2", +]: + rc, out = ssh(may_fail) + if out: + print(out) + +rc, out = ssh(f"docker compose -f {root}/compose.yml up -d act_runner_1") +if out: + print(out) +if rc != 0: + print("Runner switch failed.") + raise SystemExit(3) + +print("Phase 4: verify") +for verify_cmd in [ + "docker ps -a --format 'table {{.Names}}\t{{.Status}}' | grep -E 'NAMES|gitea-act-runner'", + "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}}'", + "docker logs --tail=30 gitea-act-runner-1 | tail -n 30", +]: + rc, out = ssh(verify_cmd) + print(out) + +rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'") +print(out if rc == 0 else "runner2-removed") + +print("One-runner rollout complete for host") +print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}") diff --git a/scripts/gitea-actions/runner-prechange-capture.xsh b/scripts/gitea-actions/runner-prechange-capture.xsh new file mode 100755 index 0000000..4609a54 --- /dev/null +++ b/scripts/gitea-actions/runner-prechange-capture.xsh @@ -0,0 +1,115 @@ +#!/usr/bin/env xonsh + +"""Capture runner restart/OOM/log baseline from all hosts before rollout changes.""" + +import os +import sys +from datetime import datetime + + +HOSTS = { + "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea", + "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", + "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", + "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", +} + +TAIL_LINES = 40 +if len(sys.argv) > 1: + try: + TAIL_LINES = int(sys.argv[1]) + except ValueError: + print("Invalid tail line count, using default 40") + +BATCH_MODE = "--batch" in sys.argv[1:] + + +def ssh(host, cmd): + if BATCH_MODE: + out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) + else: + out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) + return out.returncode, str(out.out).rstrip("\n") + + +def section(title, body): + sep = "=" * 80 + return f"\n{sep}\n{title}\n{sep}\n{body}\n" + + +report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}" +mkdir -p @(report_dir) +report_path = os.path.join(report_dir, "capture.txt") + +print("Runner pre-change capture started") +print(f"Report directory: {report_dir}") +print(f"SSH mode: {'batch' if BATCH_MODE else 'interactive'}") + +entries = {} + +for host, root in HOSTS.items(): + print(f"Collecting: {host}") + data = {"root": root} + + rc, output = ssh(host, f"test -d {root}") + data["path"] = "EXISTS" if rc == 0 else "NOT_FOUND" + + rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'") + data["ps"] = ps_text + + rc1, inspect1 = ssh( + host, + "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", + ) + rc2, inspect2 = ssh( + host, + "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", + ) + if rc1 != 0: + inspect1 = "runner1-missing" + if rc2 != 0: + inspect2 = "runner2-missing" + data["inspect"] = inspect1 + "\n" + inspect2 + + rc, oom_scan = ssh( + host, + "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'", + ) + if rc != 0: + oom_scan = "oom-scan-unavailable" + data["oom_scan"] = oom_scan + + rc, mem = ssh(host, "free -h") + data["mem"] = mem + + rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1") + rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2") + if rc1 != 0: + log1 = "runner1-logs-unavailable" + if rc2 != 0: + log2 = "runner2-logs-unavailable" + data["log1"] = log1 + data["log2"] = log2 + + entries[host] = data + +with open(report_path, "w", encoding="utf-8") as handle: + handle.write("RUNNER PRE-CHANGE CAPTURE\n") + handle.write(f"Timestamp: {datetime.now().isoformat()}\n") + handle.write(f"Tail lines: {TAIL_LINES}\n") + + for host, data in entries.items(): + handle.write("\n" + ("#" * 80) + "\n") + handle.write(f"# HOST: {host}\n") + handle.write(f"# PATH: {data['root']}\n") + handle.write(("#" * 80) + "\n") + handle.write(section("1. PATH", data.get("path", ""))) + handle.write(section("2. DOCKER PS", data.get("ps", ""))) + handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", ""))) + handle.write(section("4. OOM SCAN", data.get("oom_scan", ""))) + handle.write(section("5. MEMORY", data.get("mem", ""))) + handle.write(section("6. LOGS RUNNER1", data.get("log1", ""))) + handle.write(section("7. LOGS RUNNER2", data.get("log2", ""))) + +print("Capture complete") +print(f"Report: {report_path}")