de66e9134d
All checks were successful
CICD Start / Sanity and Base Decision (pull_request) Successful in 1m7s
Signed-off-by: copilotcoder <copilotcoder@darkhelm.org>
104 lines
3.4 KiB
Plaintext
Executable File
104 lines
3.4 KiB
Plaintext
Executable File
#!/usr/bin/env xonsh
|
|
|
|
"""Roll out one-runner stabilization on a target host (runner1 active, runner2 removed)."""
|
|
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
|
|
HOST_PATHS = {
|
|
"kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
|
|
"zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
|
"urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
|
"pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
|
}
|
|
|
|
|
|
def usage(exit_code=1):
|
|
print("Usage: xonsh rollout-one-runner.xsh <host> [--batch]")
|
|
print("Hosts:")
|
|
for host in HOST_PATHS:
|
|
print(f" - {host}")
|
|
raise SystemExit(exit_code)
|
|
|
|
|
|
if len(sys.argv) < 2:
|
|
usage()
|
|
|
|
host = sys.argv[1]
|
|
if host not in HOST_PATHS:
|
|
print(f"Unknown host: {host}")
|
|
usage()
|
|
|
|
batch_mode = "--batch" in sys.argv[2:]
|
|
root = HOST_PATHS[host]
|
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
|
|
|
|
def ssh(cmd):
|
|
if batch_mode:
|
|
out = !(ssh -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
|
|
else:
|
|
out = !(ssh -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPersist=2h -o ControlPath=~/.ssh/cm-%r@%h:%p @(host) @(cmd))
|
|
return out.returncode, str(out.out).rstrip("\n")
|
|
|
|
|
|
print(f"Host: {host}")
|
|
print(f"Path: {root}")
|
|
print(f"SSH mode: {'batch' if batch_mode else 'interactive'}")
|
|
print("Phase 1: pre-change snapshot")
|
|
|
|
rc, out = ssh(f"docker compose -f {root}/compose.yml config --services")
|
|
print(out)
|
|
|
|
rc, out = ssh("docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'")
|
|
print(out if rc == 0 else "runner1-missing")
|
|
|
|
rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}}'")
|
|
print(out if rc == 0 else "runner2-missing")
|
|
|
|
print("Phase 2: backup compose and env")
|
|
for backup_step in [
|
|
f"mkdir -p {root}/backups",
|
|
f"cp -f {root}/compose.yml {root}/backups/compose.yml.{ts}",
|
|
f"cp -f {root}/.env {root}/backups/.env.{ts}",
|
|
]:
|
|
rc, out = ssh(backup_step)
|
|
if out:
|
|
print(out)
|
|
if rc != 0:
|
|
print("Backup failed. Aborting.")
|
|
raise SystemExit(2)
|
|
print("backups-created")
|
|
|
|
print("Phase 3: switch to one runner")
|
|
for may_fail in [
|
|
f"docker compose -f {root}/compose.yml stop act_runner_2",
|
|
f"docker compose -f {root}/compose.yml rm -f act_runner_2",
|
|
]:
|
|
rc, out = ssh(may_fail)
|
|
if out:
|
|
print(out)
|
|
|
|
rc, out = ssh(f"docker compose -f {root}/compose.yml up -d act_runner_1")
|
|
if out:
|
|
print(out)
|
|
if rc != 0:
|
|
print("Runner switch failed.")
|
|
raise SystemExit(3)
|
|
|
|
print("Phase 4: verify")
|
|
for verify_cmd in [
|
|
"docker ps -a --format 'table {{.Names}}\t{{.Status}}' | grep -E 'NAMES|gitea-act-runner'",
|
|
"docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}}'",
|
|
"docker logs --tail=30 gitea-act-runner-1 | tail -n 30",
|
|
]:
|
|
rc, out = ssh(verify_cmd)
|
|
print(out)
|
|
|
|
rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'")
|
|
print(out if rc == 0 else "runner2-removed")
|
|
|
|
print("One-runner rollout complete for host")
|
|
print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}")
|