Harden runner diagnostics and warm image caches

README.md

@@ -157,9 +157,14 @@ plex-playlist/
 For Gitea Actions runner image mirror maintenance, use:
 
 - `source scripts/gitea-actions/repair_runner_mirror.xsh`
-- `source scripts/gitea-actions/check_runner_images.xsh`
+- `source scripts/gitea-actions/check_runner_images.xsh` (checks ubuntu:22.04, GHCR act image, local mirror, and Renovate image)
 - `source scripts/gitea-actions/collect_runner_diagnostics.xsh [since] [trace_id]`
 
+For setup-stage failures with sparse logs, use:
+
+- `source scripts/gitea-actions/runner-prechange-capture.xsh [tail_lines]`
+- `source scripts/gitea-actions/diagnose_runner_startup.xsh`
+
 For full troubleshooting context, see `docs/GITEA_ACTIONS_TROUBLESHOOTING.md`.
 
 ## Environment Variables

docs/GITEA_ACTIONS_TROUBLESHOOTING.md

@@ -178,7 +178,13 @@ Automation scripts for this workflow live in `scripts/gitea-actions/`:
   Repairs the mirror by pulling from GHCR, tagging/pushing to local registry, and verifying tag presence on each runner host.
 
 - `scripts/gitea-actions/check_runner_images.xsh`
-  Verifies host-by-host image presence for both upstream (`GHCR`) and mirrored (`MIRROR`) tags.
+  Verifies host-by-host image presence and pullability for `ubuntu:22.04`, upstream act image (`GHCR`), mirrored act image (`MIRROR`), and Renovate image.
+
+- `scripts/gitea-actions/runner-prechange-capture.xsh`
+  Captures restart counts, OOM flags, memory state, and runner logs across all hosts.
+
+- `scripts/gitea-actions/diagnose_runner_startup.xsh`
+  Focused startup diagnostics for runner containers and mirror pull behavior.
 
 Run from repository root (xonsh):
 

scripts/gitea-actions/check_runner_images.xsh

@@ -5,26 +5,63 @@ hosts = [
     "pi-desktop.darkhelm.lan",
 ]
 
+mirror_image = "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest"
+ghcr_runner_image = "ghcr.io/catthehacker/ubuntu:act-latest"
+renovate_image = "ghcr.io/renovatebot/renovate:41"
+python_image = "python:3.14-slim"
+node_image = "node:20-bookworm-slim"
+
 # Check both upstream and mirrored tags on each runner host.
 remote_script = """
-r = !(docker image inspect ubuntu:22.04 > /dev/null 2> /dev/null)
-print("UBUNTU22:present" if r.returncode == 0 else "UBUNTU22:missing")
-r = !(docker image inspect ghcr.io/catthehacker/ubuntu:act-latest > /dev/null 2> /dev/null)
-print("GHCR:present" if r.returncode == 0 else "GHCR:missing")
-r = !(docker image inspect kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /dev/null 2> /dev/null)
-print("MIRROR_LOCAL:present" if r.returncode == 0 else "MIRROR_LOCAL:missing")
+def local_state(label, image):
+    r = !(docker image inspect @(image) > /dev/null 2> /dev/null)
+    print(f"{label}:present" if r.returncode == 0 else f"{label}:missing")
+
+local_state("UBUNTU22", "ubuntu:22.04")
+local_state("PYTHON", "__PYTHON_IMAGE__")
+local_state("NODE", "__NODE_IMAGE__")
+local_state("GHCR", "__GHCR_RUNNER_IMAGE__")
+local_state("MIRROR_LOCAL", "__MIRROR_IMAGE__")
+local_state("RENOVATE", "__RENOVATE_IMAGE__")
 
 r = !(docker info 2> /dev/null | grep -qi "kankali.darkhelm.lan:3001")
 print("REGISTRY_CONFIG:ok" if r.returncode == 0 else "REGISTRY_CONFIG:missing")
 
-pull = !(docker pull kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest > /tmp/mirror-pull.log 2>&1)
-if pull.returncode == 0:
-    print("MIRROR_REMOTE:pull-ok")
-else:
-    mismatch = !(grep -qi "http response to https client" /tmp/mirror-pull.log)
-    print("MIRROR_REMOTE:https-mismatch" if mismatch.returncode == 0 else "MIRROR_REMOTE:pull-failed")
+def remote_pull(label, image, log_path):
+    pull = !(docker pull @(image) > @(log_path) 2>&1)
+    if pull.returncode == 0:
+        print(f"{label}:pull-ok")
+        return
+    mismatch = !(grep -qi "http response to https client" @(log_path))
+    if mismatch.returncode == 0:
+        print(f"{label}:https-mismatch")
+    else:
+        print(f"{label}:pull-failed")
+
+remote_pull("UBUNTU22_REMOTE", "ubuntu:22.04", "/tmp/ubuntu22-pull.log")
+remote_pull("PYTHON_REMOTE", "__PYTHON_IMAGE__", "/tmp/python-pull.log")
+remote_pull("NODE_REMOTE", "__NODE_IMAGE__", "/tmp/node-pull.log")
+remote_pull("GHCR_REMOTE", "__GHCR_RUNNER_IMAGE__", "/tmp/ghcr-runner-pull.log")
+remote_pull("MIRROR_REMOTE", "__MIRROR_IMAGE__", "/tmp/mirror-pull.log")
+remote_pull("RENOVATE_REMOTE", "__RENOVATE_IMAGE__", "/tmp/renovate-pull.log")
+
+local_state("UBUNTU22_AFTER_PULL", "ubuntu:22.04")
+local_state("PYTHON_AFTER_PULL", "__PYTHON_IMAGE__")
+local_state("NODE_AFTER_PULL", "__NODE_IMAGE__")
+local_state("GHCR_AFTER_PULL", "__GHCR_RUNNER_IMAGE__")
+local_state("MIRROR_AFTER_PULL", "__MIRROR_IMAGE__")
+local_state("RENOVATE_AFTER_PULL", "__RENOVATE_IMAGE__")
 """
 
+remote_script = (
+    remote_script
+    .replace("__MIRROR_IMAGE__", mirror_image)
+    .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image)
+    .replace("__RENOVATE_IMAGE__", renovate_image)
+    .replace("__PYTHON_IMAGE__", python_image)
+    .replace("__NODE_IMAGE__", node_image)
+)
+
 for host in hosts:
     print(f"\n=== {host} ===")
     ssh @(host) @(remote_script)

scripts/gitea-actions/deploy-runner-config.xsh

@@ -18,6 +18,15 @@ HOSTS = {
     "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
 }
 
+WARMUP_IMAGES = [
+    "ubuntu:22.04",
+    "python:3.14-slim",
+    "node:20-bookworm-slim",
+    "ghcr.io/catthehacker/ubuntu:act-latest",
+    "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest",
+    "ghcr.io/renovatebot/renovate:41",
+]
+
 DEFAULT_MEM_FRACTION = 0.75
 DEFAULT_JOB_MEM_FRACTION = 0.9
 DEFAULT_HOST_MEM_SHARE = 0.25
@@ -258,6 +267,20 @@ def scp(local_path, host, remote_path):
     return proc.returncode, proc.stdout + proc.stderr
 
 
+def warm_runner_images(host):
+    pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES])
+    rc, out = ssh(
+        host,
+        (
+            "set -e\n"
+            "echo 'warming_runner_images_start'\n"
+            f"{pulls}\n"
+            "echo 'warming_runner_images_done'\n"
+        ),
+    )
+    return rc, out
+
+
 def service_block_bounds(lines, start_index):
     service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" "))
     block_start = start_index + 1
@@ -743,6 +766,13 @@ with tempfile.TemporaryDirectory() as tmpdir:
             continue
         print(out.strip())
 
+        rc, out = warm_runner_images(host)
+        if rc != 0:
+            print(out.strip())
+            print("Failed to warm runner images on host")
+            continue
+        print(out.strip())
+
         rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")
         print("mounts:", out.strip())
 

scripts/gitea-actions/diagnose_runner_startup.xsh

@@ -10,6 +10,8 @@ hosts = {
 }
 
 mirror_image = 'kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest'
+ghcr_runner_image = 'ghcr.io/catthehacker/ubuntu:act-latest'
+renovate_image = 'ghcr.io/renovatebot/renovate:41'
 do_fix = '--fix' in sys.argv[1:]
 
 remote_diag = f"""
@@ -19,6 +21,9 @@ print('[runner_containers]')
 r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
 print(str(r.out).strip())
 
+r = !(docker ps -a --filter name=gitea-act-runner --format '{{{{.Names}}}}')
+runner_names = [line.strip() for line in str(r.out).splitlines() if line.strip()]
+
 print('[registry_config]')
 r = !(docker info 2> /dev/null | grep -qi 'kankali.darkhelm.lan:3001')
 print('REGISTRY_CONFIG:ok' if r.returncode == 0 else 'REGISTRY_CONFIG:missing')
@@ -36,31 +41,47 @@ else:
     tail_out = !(tail -n 20 /tmp/runner-mirror-pull.log)
     print(str(tail_out.out).strip())
 
-print('[recent_logs_runner1]')
-log1 = !(docker logs --tail 60 gitea-act-runner-1 2>&1)
-log1_lines = [
-    line for line in str(log1.out).splitlines()
-    if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
-]
-print('\\n'.join(log1_lines[-60:]))
+if not runner_names:
+    print('[recent_logs] no-runner-containers-found')
 
-print('[recent_logs_runner2]')
-log2 = !(docker logs --tail 60 gitea-act-runner-2 2>&1)
-log2_lines = [
-    line for line in str(log2.out).splitlines()
-    if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline'])
-]
-print('\\n'.join(log2_lines[-60:]))
+for name in runner_names:
+    print(f'[recent_logs_{name}]')
+    log = !(docker logs --tail 60 @(name) 2>&1)
+    log_lines = [
+        line for line in str(log.out).splitlines()
+        if any(tok in line.lower() for tok in ['error', 'failed', 'timeout', 'panic', 'unregister', 'forbidden', 'unauthorized', 'connection refused', 'context deadline', 'no route to host', 'network is unreachable'])
+    ]
+    print('\\n'.join(log_lines[-60:]))
 """
 
 remote_fix = """
 print('[fix] restarting runners')
-$(docker compose up -d --force-recreate runner1 runner2)
+r = !(docker compose up -d --force-recreate act_runner_1 act_runner_2 > /tmp/runner-restart.log 2>&1)
+if r.returncode != 0:
+    r = !(docker compose up -d --force-recreate runner1 runner2 > /tmp/runner-restart.log 2>&1)
+    if r.returncode != 0:
+        tail_out = !(tail -n 60 /tmp/runner-restart.log)
+        print(str(tail_out.out).strip())
+        raise SystemExit(1)
 $(sleep 3)
+print('[fix] warming images')
+$(docker pull ubuntu:22.04)
+$(docker pull python:3.14-slim)
+$(docker pull node:20-bookworm-slim)
+$(docker pull __GHCR_RUNNER_IMAGE__)
+$(docker pull __MIRROR_IMAGE__)
+$(docker pull __RENOVATE_IMAGE__)
 r = !(docker ps --filter name=gitea-act-runner --format '{{{{.Names}}}} {{{{.Status}}}}')
 print(str(r.out).strip())
 """
 
+remote_fix = (
+    remote_fix
+    .replace("__MIRROR_IMAGE__", mirror_image)
+    .replace("__GHCR_RUNNER_IMAGE__", ghcr_runner_image)
+    .replace("__RENOVATE_IMAGE__", renovate_image)
+)
+
 for host, path in hosts.items():
     print(f"\n===== {host} =====")
     cmd = f"cd {path}\n{remote_diag}"

scripts/gitea-actions/rollout-one-runner.xsh

@@ -13,6 +13,15 @@ HOST_PATHS = {
     "pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
 }
 
+WARMUP_IMAGES = [
+    "ubuntu:22.04",
+    "python:3.14-slim",
+    "node:20-bookworm-slim",
+    "ghcr.io/catthehacker/ubuntu:act-latest",
+    "kankali.darkhelm.lan:3001/darkhelm.org/act-ubuntu:act-latest",
+    "ghcr.io/renovatebot/renovate:41",
+]
+
 
 def usage(exit_code=1):
     print("Usage: xonsh rollout-one-runner.xsh <host> [--batch]")
@@ -99,5 +108,14 @@ for verify_cmd in [
 rc, out = ssh("docker inspect gitea-act-runner-2 --format 'runner2 status={{.State.Status}}'")
 print(out if rc == 0 else "runner2-removed")
 
+print("Phase 5: warm images")
+pulls = "\n".join([f"docker pull {image}" for image in WARMUP_IMAGES])
+rc, out = ssh(f"set -e\n{pulls}")
+if rc != 0:
+    print("Warmup failed")
+    print(out)
+    raise SystemExit(4)
+print(out)
+
 print("One-runner rollout complete for host")
 print(f"Backup files: {root}/backups/compose.yml.{ts} and {root}/backups/.env.{ts}")

scripts/gitea-actions/runner-prechange-capture.xsh

@@ -37,6 +37,16 @@ def section(title, body):
     return f"\n{sep}\n{title}\n{sep}\n{body}\n"
 
 
+def list_runner_containers(host):
+    rc, text = ssh(
+        host,
+        "docker ps -a --filter name=gitea-act-runner --format '{{.Names}}'",
+    )
+    if rc != 0 or not text:
+        return []
+    return [line.strip() for line in text.splitlines() if line.strip()]
+
+
 report_dir = f"/tmp/runner-prechange-{datetime.now().strftime('%Y%m%d_%H%M%S')}"
 mkdir -p @(report_dir)
 report_path = os.path.join(report_dir, "capture.txt")
@@ -57,39 +67,45 @@ for host, root in HOSTS.items():
     rc, ps_text = ssh(host, "docker ps -a --format 'table {{.Names}}\t{{.Status}}'")
     data["ps"] = ps_text
 
-    rc1, inspect1 = ssh(
-        host,
-        "docker inspect gitea-act-runner-1 --format 'runner1 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
-    )
-    rc2, inspect2 = ssh(
-        host,
-        "docker inspect gitea-act-runner-2 --format 'runner2 restart={{.RestartCount}} oom={{.State.OOMKilled}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}'",
-    )
-    if rc1 != 0:
-        inspect1 = "runner1-missing"
-    if rc2 != 0:
-        inspect2 = "runner2-missing"
-    data["inspect"] = inspect1 + "\n" + inspect2
+    runner_containers = list_runner_containers(host)
+    data["runner_containers"] = runner_containers
 
-    rc, oom_scan = ssh(
-        host,
-        "docker inspect gitea-act-runner-1 gitea-act-runner-2 --format '{{.Name}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}} error={{.State.Error}}'",
-    )
-    if rc != 0:
-        oom_scan = "oom-scan-unavailable"
-    data["oom_scan"] = oom_scan
+    inspect_lines = []
+    oom_lines = []
+    if not runner_containers:
+        inspect_lines.append("no-runner-containers-found")
+        oom_lines.append("no-runner-containers-found")
+
+    for container in runner_containers:
+        rc, inspect_text = ssh(
+            host,
+            f"docker inspect {container} --format '{container} restart={{{{.RestartCount}}}} oom={{{{.State.OOMKilled}}}} status={{{{.State.Status}}}} started={{{{.State.StartedAt}}}} finished={{{{.State.FinishedAt}}}}'",
+        )
+        if rc != 0:
+            inspect_text = f"{container}-inspect-unavailable"
+        inspect_lines.append(inspect_text)
+
+        rc, oom_text = ssh(
+            host,
+            f"docker inspect {container} --format '{{{{.Name}}}} oom={{{{.State.OOMKilled}}}} exit={{{{.State.ExitCode}}}} error={{{{.State.Error}}}}'",
+        )
+        if rc != 0:
+            oom_text = f"{container}-oom-scan-unavailable"
+        oom_lines.append(oom_text)
+
+    data["inspect"] = "\n".join(inspect_lines)
+    data["oom_scan"] = "\n".join(oom_lines)
 
     rc, mem = ssh(host, "free -h")
     data["mem"] = mem
 
-    rc1, log1 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-1")
-    rc2, log2 = ssh(host, f"docker logs --tail={TAIL_LINES} gitea-act-runner-2")
-    if rc1 != 0:
-        log1 = "runner1-logs-unavailable"
-    if rc2 != 0:
-        log2 = "runner2-logs-unavailable"
-    data["log1"] = log1
-    data["log2"] = log2
+    runner_logs = {}
+    for container in runner_containers:
+        rc, log_text = ssh(host, f"docker logs --tail={TAIL_LINES} {container}")
+        if rc != 0:
+            log_text = f"{container}-logs-unavailable"
+        runner_logs[container] = log_text
+    data["runner_logs"] = runner_logs
 
     entries[host] = data
 
@@ -108,8 +124,12 @@ with open(report_path, "w", encoding="utf-8") as handle:
         handle.write(section("3. INSPECT RESTART/OOM", data.get("inspect", "")))
         handle.write(section("4. OOM SCAN", data.get("oom_scan", "")))
         handle.write(section("5. MEMORY", data.get("mem", "")))
-        handle.write(section("6. LOGS RUNNER1", data.get("log1", "")))
-        handle.write(section("7. LOGS RUNNER2", data.get("log2", "")))
+        containers = data.get("runner_containers", [])
+        if not containers:
+            handle.write(section("6. RUNNER LOGS", "no-runner-containers-found"))
+        else:
+            for idx, container in enumerate(containers, start=6):
+                handle.write(section(f"{idx}. RUNNER LOGS ({container})", data.get("runner_logs", {}).get(container, "")))
 
 print("Capture complete")
 print(f"Report: {report_path}")