From 9731fee9bea3d2a5d8d1072702d2a457c6326188 Mon Sep 17 00:00:00 2001 From: copilotcoder Date: Thu, 9 Jul 2026 13:16:16 -0400 Subject: [PATCH] Harden runner diagnostics and warm image caches --- README.md | 7 +- docs/GITEA_ACTIONS_TROUBLESHOOTING.md | 8 +- scripts/gitea-actions/check_runner_images.xsh | 61 +++++++++++--- .../gitea-actions/deploy-runner-config.xsh | 30 +++++++ .../gitea-actions/diagnose_runner_startup.xsh | 51 ++++++++---- scripts/gitea-actions/rollout-one-runner.xsh | 18 +++++ .../runner-prechange-capture.xsh | 80 ++++++++++++------- 7 files changed, 196 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index eac02b2..d9eb8b1 100644 --- a/README.md +++ b/README.md @@ -157,9 +157,14 @@ plex-playlist/ For Gitea Actions runner image mirror maintenance, use: - `source scripts/gitea-actions/repair_runner_mirror.xsh` -- `source scripts/gitea-actions/check_runner_images.xsh` +- `source scripts/gitea-actions/check_runner_images.xsh` (checks ubuntu:22.04, GHCR act image, local mirror, and Renovate image) - `source scripts/gitea-actions/collect_runner_diagnostics.xsh [since] [trace_id]` +For setup-stage failures with sparse logs, use: + +- `source scripts/gitea-actions/runner-prechange-capture.xsh [tail_lines]` +- `source scripts/gitea-actions/diagnose_runner_startup.xsh` + For full troubleshooting context, see `docs/GITEA_ACTIONS_TROUBLESHOOTING.md`. ## Environment Variables diff --git a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md index 54803c3..15bc1c0 100644 --- a/docs/GITEA_ACTIONS_TROUBLESHOOTING.md +++ b/docs/GITEA_ACTIONS_TROUBLESHOOTING.md @@ -178,7 +178,13 @@ Automation scripts for this workflow live in `scripts/gitea-actions/`: Repairs the mirror by pulling from GHCR, tagging/pushing to local registry, and verifying tag presence on each runner host. - `scripts/gitea-actions/check_runner_images.xsh` - Verifies host-by-host image presence for both upstream (`GHCR`) and mirrored (`MIRROR`) tags. + Verifies host-by-host image presence and pullability for `ubuntu:22.04`, upstream act image (`GHCR`), mirrored act image (`MIRROR`), and Renovate image. + +- `scripts/gitea-actions/runner-prechange-capture.xsh` + Captures restart counts, OOM flags, memory state, and runner logs across all hosts. + +- `scripts/gitea-actions/diagnose_runner_startup.xsh` + Focused startup diagnostics for runner containers and mirror pull behavior. Run from repository root (xonsh): diff --git a/scripts/gitea-actions/check_runner_images.xsh b/scripts/gitea-actions/check_runner_images.xsh index 5432232..c1d43e6 100644 --- a/scripts/gitea-actions/check_runner_images.xsh +++ b/scripts/gitea-actions/check_runner_images.xsh @@ -5,26 +5,63 @@ hosts = [ "pi-desktop.darkhelm.lan", ] +mirror_image = "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest" +ghcr_runner_image = "ghcr.io/catthehacker/ubuntu:act-latest" +renovate_image = "ghcr.io/renovatebot/renovate:41" +python_image = "python:3.14-slim" +node_image = "node:20-bookworm-slim" + # Check both upstream and mirrored tags on each runner host. remote_script = """ -r = !(docker image inspect ubuntu:22.04 > /dev/null 2> /dev/null) -print("UBUNTU22:present" if r.returncode == 0 else "UBUNTU22:missing") -r = !(docker image inspect ghcr.io/catthehacker/ubuntu:act-latest > /dev/null 2> /dev/null) -print("GHCR:present" if r.returncode == 0 else "GHCR:missing") -r = !(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null) -print("MIRROR_LOCAL:present" if r.returncode == 0 else "MIRROR_LOCAL:missing") +def local_state(label, image): + r = !(docker image inspect @(image) > /dev/null 2> /dev/null) + print(f"{label}:present" if r.returncode == 0 else f"{label}:missing") + +local_state("UBUNTU22", "ubuntu:22.04") +local_state("PYTHON", "__PYTHON_IMAGE__") +local_state("NODE", "__NODE_IMAGE__") +local_state("GHCR", "__GHCR_RUNNER_IMAGE__") +local_state("MIRROR_LOCAL", "__MIRROR_IMAGE__") +local_state("RENOVATE", "__RENOVATE_IMAGE__") r = !(docker info 2> /dev/null | grep -qi "kankali.darkhelm.lan:3001") print("REGISTRY_CONFIG:ok" if r.returncode == 0 else "REGISTRY_CONFIG:missing") -pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-pull.log 2>&1) -if pull.returncode == 0: - print("MIRROR_REMOTE:pull-ok") -else: - mismatch = !(grep -qi "http response to https client" /tmp/mirror-pull.log) - print("MIRROR_REMOTE:https-mismatch" if mismatch.returncode == 0 else "MIRROR_REMOTE:pull-failed") +def remote_pull(label, image, log_path): + pull = !(docker pull @(image) > @(log_path) 2>&1) + if pull.returncode == 0: + print(f"{label}:pull-ok") + return + mismatch = !(grep -qi "http response to https client" @(log_path)) + if mismatch.returncode == 0: + print(f"{label}:https-mismatch") + else: + print(f"{label}:pull-failed") + +remote_pull("UBUNTU22_REMOTE", "ubuntu:22.04", "/tmp/ubuntu22-pull.log") +remote_pull("PYTHON_REMOTE", "__PYTHON_IMAGE__", "/tmp/python-pull.log") +remote_pull("NODE_REMOTE", "__NODE_IMAGE__", "/tmp/node-pull.log") +remote_pull("GHCR_REMOTE", "__GHCR_RUNNER_IMAGE__", "/tmp/ghcr-runner-pull.log") +remote_pull("MIRROR_REMOTE", "__MIRROR_IMAGE__", "/tmp/mirror-pull.log") +remote_pull("RENOVATE_REMOTE", "__RENOVATE_IMAGE__", "/tmp/renovate-pull.log") + +local_state("UBUNTU22_AFTER_PULL", "ubuntu:22.04") +local_state("PYTHON_AFTER_PULL", "__PYTHON_IMAGE__") +local_state("NODE_AFTER_PULL", "__NODE_IMAGE__") +local_state("GHCR_AFTER_PULL", "__GHCR_RUNNER_IMAGE__") +local_state("MIRROR_AFTER_PULL", "__MIRROR_IMAGE__") +local_state("RENOVATE_AFTER_PULL", "__RENOVATE_IMAGE__") """ +remote_script = ( + remote_script + .replace("__MIRROR_IMAGE__", mirror_image) + .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image) + .replace("__RENOVATE_IMAGE__", renovate_image) + .replace("__PYTHON_IMAGE__", python_image) + .replace("__NODE_IMAGE__", node_image) +) + for host in hosts: print(f"\n=== {host} ===") ssh @(host) @(remote_script) diff --git a/scripts/gitea-actions/deploy-runner-config.xsh b/scripts/gitea-actions/deploy-runner-config.xsh index 8186902..8cc067c 100755 --- a/scripts/gitea-actions/deploy-runner-config.xsh +++ b/scripts/gitea-actions/deploy-runner-config.xsh @@ -18,6 +18,15 @@ HOSTS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +WARMUP_IMAGES = [ + "ubuntu:22.04", + "python:3.14-slim", + "node:20-bookworm-slim", + "ghcr.io/catthehacker/ubuntu:act-latest", + "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", + "ghcr.io/renovatebot/renovate:41", +] + DEFAULT_MEM_FRACTION = 0.75 DEFAULT_JOB_MEM_FRACTION = 0.9 DEFAULT_HOST_MEM_SHARE = 0.25 @@ -258,6 +267,20 @@ def scp(local_path, host, remote_path): return proc.returncode, proc.stdout + proc.stderr +def warm_runner_images(host): + pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) + rc, out = ssh( + host, + ( + "set -e\n" + "echo 'warming_runner_images_start'\n" + f"{pulls}\n" + "echo 'warming_runner_images_done'\n" + ), + ) + return rc, out + + def service_block_bounds(lines, start_index): service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" ")) block_start = start_index + 1 @@ -743,6 +766,13 @@ with tempfile.TemporaryDirectory() as tmpdir: continue print(out.strip()) + rc, out = warm_runner_images(host) + if rc != 0: + print(out.strip()) + print("Failed to warm runner images on host") + continue + print(out.strip()) + rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'") print("mounts:", out.strip()) diff --git a/scripts/gitea-actions/diagnose_runner_startup.xsh b/scripts/gitea-actions/diagnose_runner_startup.xsh index 8339c98..70ec106 100755 --- a/scripts/gitea-actions/diagnose_runner_startup.xsh +++ b/scripts/gitea-actions/diagnose_runner_startup.xsh @@ -10,6 +10,8 @@ hosts = { } mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest' +ghcr_runner_image = 'ghcr.io/catthehacker/ubuntu:act-latest' +renovate_image = 'ghcr.io/renovatebot/renovate:41' do_fix = '--fix' in sys.argv[1:] remote_diag = f""" @@ -19,6 +21,9 @@ print('[runner_containers]') r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') print(str(r.out).strip()) +r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}}') +runner_names = [line.strip() for line in str(r.out).splitlines() if line.strip()] + print('[registry_config]') r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001') print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing') @@ -36,31 +41,47 @@ else: tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log) print(str(tail_out.out).strip()) -print('[recent_logs_runner1]') -log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1) -log1_lines = [ - line for line in str(log1.out).splitlines() - if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) -] -print('\\n'.join(log1_lines[-60:])) +if not runner_names: + print('[recent_logs] no-runner-containers-found') -print('[recent_logs_runner2]') -log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1) -log2_lines = [ - line for line in str(log2.out).splitlines() - if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline']) -] -print('\\n'.join(log2_lines[-60:])) +for name in runner_names: + print(f'[recent_logs_{name}]') + log = !(docker logs --tail 60 @(name) 2>&1) + log_lines = [ + line for line in str(log.out).splitlines() + if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline', 'no route to host', 'network is unreachable']) + ] + print('\\n'.join(log_lines[-60:])) """ remote_fix = """ print('[fix] restarting runners') -$(docker compose up -d --force-recreate runner1 runner2) +r = !(docker compose up -d --force-recreate act_runner_1 act_runner_2 > /tmp/runner-restart.log 2>&1) +if r.returncode != 0: + r = !(docker compose up -d --force-recreate runner1 runner2 > /tmp/runner-restart.log 2>&1) + if r.returncode != 0: + tail_out = !(tail -n 60 /tmp/runner-restart.log) + print(str(tail_out.out).strip()) + raise SystemExit(1) $(sleep 3) +print('[fix] warming images') +$(docker pull ubuntu:22.04) +$(docker pull python:3.14-slim) +$(docker pull node:20-bookworm-slim) +$(docker pull __GHCR_RUNNER_IMAGE__) +$(docker pull __MIRROR_IMAGE__) +$(docker pull __RENOVATE_IMAGE__) r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}') print(str(r.out).strip()) """ +remote_fix = ( + remote_fix + .replace("__MIRROR_IMAGE__", mirror_image) + .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image) + .replace("__RENOVATE_IMAGE__", renovate_image) +) + for host, path in hosts.items(): print(f"\n===== {host} =====") cmd = f"cd {path}\n{remote_diag}" diff --git a/scripts/gitea-actions/rollout-one-runner.xsh b/scripts/gitea-actions/rollout-one-runner.xsh index afb5578..c97a8ef 100755 --- a/scripts/gitea-actions/rollout-one-runner.xsh +++ b/scripts/gitea-actions/rollout-one-runner.xsh @@ -13,6 +13,15 @@ HOST_PATHS = { "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner", } +WARMUP_IMAGES = [ + "ubuntu:22.04", + "python:3.14-slim", + "node:20-bookworm-slim", + "ghcr.io/catthehacker/ubuntu:act-latest", + "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest", + "ghcr.io/renovatebot/renovate:41", +] + def usage(exit_code=1): print("Usage: xonsh rollout-one-runner.xsh [--batch]") @@ -99,5 +108,14 @@ for verify_cmd in [ rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'") print(out if rc == 0 else "runner2-removed") +print("Phase 5: warm images") +pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES]) +rc, out = ssh(f"set -e\n{pulls}") +if rc != 0: + print("Warmup failed") + print(out) + raise SystemExit(4) +print(out) + print("One-runner rollout complete for host") print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}") diff --git a/scripts/gitea-actions/runner-prechange-capture.xsh b/scripts/gitea-actions/runner-prechange-capture.xsh index 4609a54..527bd9b 100755 --- a/scripts/gitea-actions/runner-prechange-capture.xsh +++ b/scripts/gitea-actions/runner-prechange-capture.xsh @@ -37,6 +37,16 @@ def section(title, body): return f"\n{sep}\n{title}\n{sep}\n{body}\n" +def list_runner_containers(host): + rc, text = ssh( + host, + "docker ps -a --filter name=gitea-act-runner --format '{{.Names}}'", + ) + if rc != 0 or not text: + return [] + return [line.strip() for line in text.splitlines() if line.strip()] + + report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}" mkdir -p @(report_dir) report_path = os.path.join(report_dir, "capture.txt") @@ -57,39 +67,45 @@ for host, root in HOSTS.items(): rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'") data["ps"] = ps_text - rc1, inspect1 = ssh( - host, - "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", - ) - rc2, inspect2 = ssh( - host, - "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'", - ) - if rc1 != 0: - inspect1 = "runner1-missing" - if rc2 != 0: - inspect2 = "runner2-missing" - data["inspect"] = inspect1 + "\n" + inspect2 + runner_containers = list_runner_containers(host) + data["runner_containers"] = runner_containers - rc, oom_scan = ssh( - host, - "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'", - ) - if rc != 0: - oom_scan = "oom-scan-unavailable" - data["oom_scan"] = oom_scan + inspect_lines = [] + oom_lines = [] + if not runner_containers: + inspect_lines.append("no-runner-containers-found") + oom_lines.append("no-runner-containers-found") + + for container in runner_containers: + rc, inspect_text = ssh( + host, + f"docker inspect {container} --format '{container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}'", + ) + if rc != 0: + inspect_text = f"{container}-inspect-unavailable" + inspect_lines.append(inspect_text) + + rc, oom_text = ssh( + host, + f"docker inspect {container} --format '{{{{.Name}}}} oom={{{{.State.OOMKilled}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}'", + ) + if rc != 0: + oom_text = f"{container}-oom-scan-unavailable" + oom_lines.append(oom_text) + + data["inspect"] = "\n".join(inspect_lines) + data["oom_scan"] = "\n".join(oom_lines) rc, mem = ssh(host, "free -h") data["mem"] = mem - rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1") - rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2") - if rc1 != 0: - log1 = "runner1-logs-unavailable" - if rc2 != 0: - log2 = "runner2-logs-unavailable" - data["log1"] = log1 - data["log2"] = log2 + runner_logs = {} + for container in runner_containers: + rc, log_text = ssh(host, f"docker logs --tail={TAIL_LINES} {container}") + if rc != 0: + log_text = f"{container}-logs-unavailable" + runner_logs[container] = log_text + data["runner_logs"] = runner_logs entries[host] = data @@ -108,8 +124,12 @@ with open(report_path, "w", encoding="utf-8") as handle: handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", ""))) handle.write(section("4. OOM SCAN", data.get("oom_scan", ""))) handle.write(section("5. MEMORY", data.get("mem", ""))) - handle.write(section("6. LOGS RUNNER1", data.get("log1", ""))) - handle.write(section("7. LOGS RUNNER2", data.get("log2", ""))) + containers = data.get("runner_containers", []) + if not containers: + handle.write(section("6. RUNNER LOGS", "no-runner-containers-found")) + else: + for idx, container in enumerate(containers, start=6): + handle.write(section(f"{idx}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, ""))) print("Capture complete") print(f"Report: {report_path}")