plex-playlist/scripts/gitea-actions/diagnose_runner_startup.xsh

#!/usr/bin/env xonsh

import sys

hosts = {
    'kankali.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea',
    'zhokq.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
    'urtzul.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
    'pi-desktop.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
}

mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest'
ghcr_runner_image = 'ghcr.io/catthehacker/ubuntu:act-latest'
renovate_image = 'ghcr.io/renovatebot/renovate:41'
do_fix = '--fix' in sys.argv[1:]

remote_diag = f"""
print('pwd=' + $(pwd).strip())

print('[runner_containers]')
r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
print(str(r.out).strip())

r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}}')
runner_names = [line.strip() for line in str(r.out).splitlines() if line.strip()]

print('[registry_config]')
r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001')
print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing')

print('[image_state]')
r = !(docker image inspect {mirror_image} > /dev/null 2> /dev/null)
print('MIRROR_LOCAL:present' if r.returncode == 0 else 'MIRROR_LOCAL:missing')

r = !(docker pull {mirror_image} > /tmp/runner-mirror-pull.log 2>&1)
if r.returncode == 0:
    print('MIRROR_REMOTE:pull-ok')
else:
    mismatch = !(grep -qi 'http response to https client' /tmp/runner-mirror-pull.log)
    print('MIRROR_REMOTE:https-mismatch' if mismatch.returncode == 0 else 'MIRROR_REMOTE:pull-failed')
    tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log)
    print(str(tail_out.out).strip())

if not runner_names:
    print('[recent_logs] no-runner-containers-found')

for name in runner_names:
    print(f'[recent_logs_{name}]')
    log = !(docker logs --tail 60 @(name) 2>&1)
    log_lines = [
        line for line in str(log.out).splitlines()
        if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline', 'no route to host', 'network is unreachable'])
    ]
    print('\\n'.join(log_lines[-60:]))
"""

remote_fix = """
print('[fix] restarting runners')
r = !(docker compose up -d --force-recreate act_runner_1 act_runner_2 > /tmp/runner-restart.log 2>&1)
if r.returncode != 0:
    r = !(docker compose up -d --force-recreate runner1 runner2 > /tmp/runner-restart.log 2>&1)
    if r.returncode != 0:
        tail_out = !(tail -n 60 /tmp/runner-restart.log)
        print(str(tail_out.out).strip())
        raise SystemExit(1)
$(sleep 3)
print('[fix] warming images')
$(docker pull ubuntu:22.04)
$(docker pull python:3.14-slim)
$(docker pull node:20-bookworm-slim)
$(docker pull __GHCR_RUNNER_IMAGE__)
$(docker pull __MIRROR_IMAGE__)
$(docker pull __RENOVATE_IMAGE__)
r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
print(str(r.out).strip())
"""

remote_fix = (
    remote_fix
    .replace("__MIRROR_IMAGE__", mirror_image)
    .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image)
    .replace("__RENOVATE_IMAGE__", renovate_image)
)

for host, path in hosts.items():
    print(f"\n===== {host} =====")
    cmd = f"cd {path}\n{remote_diag}"
    r = !(ssh @(host) @(cmd))
    if r.returncode != 0:
        print('DIAG_FAILED')
        print(str(r.out).strip())
        continue

    print(str(r.out).strip())

    if do_fix:
        print('[fix] applying restart')
        fix_cmd = f"cd {path}\n{remote_fix}"
        f = !(ssh @(host) @(fix_cmd))
        if f.returncode != 0:
            print('FIX_FAILED')
            print(str(f.out).strip())
        else:
            print(str(f.out).strip())