de66e9134d
All checks were successful
CICD Start / Sanity and Base Decision (pull_request) Successful in 1m7s
Signed-off-by: copilotcoder <copilotcoder@darkhelm.org>
84 lines
2.9 KiB
Plaintext
Executable File
84 lines
2.9 KiB
Plaintext
Executable File
#!/usr/bin/env xonsh
|
|
|
|
import sys
|
|
|
|
hosts = {
|
|
'kankali.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea',
|
|
'zhokq.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
|
|
'urtzul.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
|
|
'pi-desktop.darkhelm.lan': '/home/darkhelm/Projects/DarkHelm.org/gitea-runner',
|
|
}
|
|
|
|
mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest'
|
|
do_fix = '--fix' in sys.argv[1:]
|
|
|
|
remote_diag = f"""
|
|
print('pwd=' + $(pwd).strip())
|
|
|
|
print('[runner_containers]')
|
|
r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
|
|
print(str(r.out).strip())
|
|
|
|
print('[registry_config]')
|
|
r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001')
|
|
print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing')
|
|
|
|
print('[image_state]')
|
|
r = !(docker image inspect {mirror_image} > /dev/null 2> /dev/null)
|
|
print('MIRROR_LOCAL:present' if r.returncode == 0 else 'MIRROR_LOCAL:missing')
|
|
|
|
r = !(docker pull {mirror_image} > /tmp/runner-mirror-pull.log 2>&1)
|
|
if r.returncode == 0:
|
|
print('MIRROR_REMOTE:pull-ok')
|
|
else:
|
|
mismatch = !(grep -qi 'http response to https client' /tmp/runner-mirror-pull.log)
|
|
print('MIRROR_REMOTE:https-mismatch' if mismatch.returncode == 0 else 'MIRROR_REMOTE:pull-failed')
|
|
tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log)
|
|
print(str(tail_out.out).strip())
|
|
|
|
print('[recent_logs_runner1]')
|
|
log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1)
|
|
log1_lines = [
|
|
line for line in str(log1.out).splitlines()
|
|
if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
|
|
]
|
|
print('\\n'.join(log1_lines[-60:]))
|
|
|
|
print('[recent_logs_runner2]')
|
|
log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1)
|
|
log2_lines = [
|
|
line for line in str(log2.out).splitlines()
|
|
if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
|
|
]
|
|
print('\\n'.join(log2_lines[-60:]))
|
|
"""
|
|
|
|
remote_fix = """
|
|
print('[fix] restarting runners')
|
|
$(docker compose up -d --force-recreate runner1 runner2)
|
|
$(sleep 3)
|
|
r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
|
|
print(str(r.out).strip())
|
|
"""
|
|
|
|
for host, path in hosts.items():
|
|
print(f"\n===== {host} =====")
|
|
cmd = f"cd {path}\n{remote_diag}"
|
|
r = !(ssh @(host) @(cmd))
|
|
if r.returncode != 0:
|
|
print('DIAG_FAILED')
|
|
print(str(r.out).strip())
|
|
continue
|
|
|
|
print(str(r.out).strip())
|
|
|
|
if do_fix:
|
|
print('[fix] applying restart')
|
|
fix_cmd = f"cd {path}\n{remote_fix}"
|
|
f = !(ssh @(host) @(fix_cmd))
|
|
if f.returncode != 0:
|
|
print('FIX_FAILED')
|
|
print(str(f.out).strip())
|
|
else:
|
|
print(str(f.out).strip())
|