#!/usr/bin/env xonsh """Roll out one-runner stabilization on a target host (runner1 active, runner2 removed).""" import sys from datetime import datetime HOST_PATHS = { "kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea", "zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", "urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } WARMUP_IMAGES = [ "ubuntu:22.04", "python:3.14-slim", "node:20-bookworm-slim", "ghcr.io/catthehacker/ubuntu:act-latest", "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", "ghcr.io/renovatebot/renovate:41", ] def usage(exit_code=1): print("Usage: xonsh rollout-one-runner.xsh [--batch]") print("Hosts:") for host in HOST_PATHS: print(f" - {host}") raise SystemExit(exit_code) if len(sys.argv) < 2: usage() host = sys.argv[1] if host not in HOST_PATHS: print(f"Unknown host: {host}") usage() batch_mode = "--batch" in sys.argv[2:] root = HOST_PATHS[host] ts = datetime.now().strftime("%Y%m%d_%H%M%S") def ssh(cmd): if batch_mode: out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) else: out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd)) return out.returncode, str(out.out).rstrip("\n") print(f"Host: {host}") print(f"Path: {root}") print(f"SSH mode: {'batch' if batch_mode else 'interactive'}") print("Phase 1: pre-change snapshot") rc, out = ssh(f"docker compose -f {root}/compose.yml config --services") print(out) rc, out = ssh("docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'") print(out if rc == 0 else "runner1-missing") rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'") print(out if rc == 0 else "runner2-missing") print("Phase 2: backup compose and env") for backup_step in [ f"mkdir -p {root}/backups", f"cp -f {root}/compose.yml {root}/backups/compose.yml.{ts}", f"cp -f {root}/.env {root}/backups/.env.{ts}", ]: rc, out = ssh(backup_step) if out: print(out) if rc != 0: print("Backup failed. Aborting.") raise SystemExit(2) print("backups-created") print("Phase 3: switch to one runner") for may_fail in [ f"docker compose -f {root}/compose.yml stop act_runner_2", f"docker compose -f {root}/compose.yml rm -f act_runner_2", ]: rc, out = ssh(may_fail) if out: print(out) rc, out = ssh(f"docker compose -f {root}/compose.yml up -d act_runner_1") if out: print(out) if rc != 0: print("Runner switch failed.") raise SystemExit(3) print("Phase 4: verify") for verify_cmd in [ "docker ps -a --format 'table {{.Names}}\t{{.Status}}' | grep -E 'NAMES|gitea-act-runner'", "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}}'", "docker logs --tail=30 gitea-act-runner-1 | tail -n 30", ]: rc, out = ssh(verify_cmd) print(out) rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'") print(out if rc == 0 else "runner2-removed") print("Phase 5: warm images") pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) rc, out = ssh(f"set -e\n{pulls}") if rc != 0: print("Warmup failed") print(out) raise SystemExit(4) print(out) print("One-runner rollout complete for host") print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}")