Xlorep DarkHelm
1f6cafa1bc
## Summary
Replace the existing source-context integration lane with backend runtime black-box integration checks that run against started deployable containers.
This change wires deployable backend image references (both commit tag and immutable digest) from the build workflow into the tests workflow, then validates runtime behavior over network endpoints.
## Why
Integration confidence should come from testing running service artifacts, not only source-mounted or in-process execution.
## What Changed
- Build workflow now:
- Publishes deployable backend image tag reference and digest reference
- Exposes both as job outputs
- Passes both references into CICD Tests dispatch inputs
- CICD Tests workflow now:
- Accepts deployable backend tag and digest inputs
- Propagates these through setup outputs
- Replaces previous integration lane behavior with runtime black-box execution:
- Starts isolated Docker network
- Starts Postgres container
- Starts backend container from digest-pinned deployable image
- Enforces tag-to-digest consistency before running checks
- Runs endpoint checks against live container:
- GET /
- GET /compatibility
- GET /health
- Captures backend/db logs and container state on failure
- Cleans up containers and network via trap
- Documentation updated:
- Runtime contract enforcement section now includes runtime black-box integration checks
- CI success summary now reflects runtime integration lane behavior
## Scope
Included:
- Backend runtime black-box integration replacement for the existing integration lane
- Digest + tag identity enforcement
- Failure diagnostics for triage
Out of scope:
- Frontend runtime smoke checks
- E2E lane redesign
## Acceptance Criteria Mapping
- Integration tests execute against runtime container endpoints: ✅
- Integration lane consumes built image references (not source-mounted execution): ✅
- Failures surface service logs and test logs for triage: ✅
## Verification
- Workflow files pass local validation checks
- Pre-commit hooks pass on committed changes
- Branch pushed and ready for PR review
## Related
- Issue: #61
- Dependency context: #66
Co-authored-by: copilotcoder <copilotcoder@darkhelm.org>
Reviewed-on: #72
752 lines
25 KiB
Plaintext
Executable File
752 lines
25 KiB
Plaintext
Executable File
#!/usr/bin/env xonsh
|
|
|
|
"""Deploy shared runner config.yaml to all hosts and wire compose to use CONFIG_FILE."""
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from datetime import datetime
|
|
|
|
TEMPLATE_HOST = "kankali.darkhelm.lan"
|
|
TEMPLATE_PATH = "/home/darkhelm/Projects/DarkHelm.org/gitea/runner-config.template.yaml"
|
|
|
|
HOSTS = {
|
|
"kankali.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea",
|
|
"zhokq.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
|
"urtzul.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
|
"pi-desktop.darkhelm.lan": "/home/darkhelm/Projects/DarkHelm.org/gitea-runner",
|
|
}
|
|
|
|
DEFAULT_MEM_FRACTION = 0.75
|
|
DEFAULT_JOB_MEM_FRACTION = 0.9
|
|
DEFAULT_HOST_MEM_SHARE = 0.25
|
|
DEFAULT_HOST_CPU_SHARE = 0.25
|
|
DEFAULT_CPU_FRACTION = 1.0
|
|
DEFAULT_JOB_CPU_FRACTION = 0.9
|
|
DEFAULT_RUNNER_MEM_SWAP_FACTOR = 1.0
|
|
DEFAULT_JOB_MEM_SWAP_FACTOR = 1.0
|
|
DEFAULT_RUNNER_PIDS_LIMIT = 1024
|
|
DEFAULT_JOB_PIDS_LIMIT = 512
|
|
DEFAULT_JOB_NOFILE_SOFT = 2048
|
|
DEFAULT_JOB_NOFILE_HARD = 4096
|
|
DEFAULT_RUNNER_MAX_PARALLEL_JOBS = 1
|
|
MIN_RUNNER_MEM_MIB = 256
|
|
MIN_JOB_MEM_MIB = 128
|
|
|
|
|
|
def parse_mem_fraction(argv):
|
|
fraction = DEFAULT_MEM_FRACTION
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--mem-fraction="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
fraction = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --mem-fraction value: {value}") from exc
|
|
if fraction <= 0 or fraction > 1:
|
|
raise SystemExit("--mem-fraction must be > 0 and <= 1")
|
|
return fraction
|
|
|
|
|
|
def parse_job_mem_fraction(argv):
|
|
fraction = DEFAULT_JOB_MEM_FRACTION
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--job-mem-fraction="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
fraction = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --job-mem-fraction value: {value}") from exc
|
|
if fraction <= 0 or fraction > 1:
|
|
raise SystemExit("--job-mem-fraction must be > 0 and <= 1")
|
|
return fraction
|
|
|
|
|
|
def parse_host_mem_share(argv):
|
|
share = DEFAULT_HOST_MEM_SHARE
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--host-mem-share="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
share = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --host-mem-share value: {value}") from exc
|
|
if share <= 0 or share > 1:
|
|
raise SystemExit("--host-mem-share must be > 0 and <= 1")
|
|
return share
|
|
|
|
|
|
def parse_host_cpu_share(argv):
|
|
share = DEFAULT_HOST_CPU_SHARE
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--host-cpu-share="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
share = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --host-cpu-share value: {value}") from exc
|
|
if share <= 0 or share > 1:
|
|
raise SystemExit("--host-cpu-share must be > 0 and <= 1")
|
|
return share
|
|
|
|
|
|
def parse_cpu_fraction(argv):
|
|
fraction = DEFAULT_CPU_FRACTION
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--cpu-fraction="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
fraction = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --cpu-fraction value: {value}") from exc
|
|
if fraction <= 0 or fraction > 1:
|
|
raise SystemExit("--cpu-fraction must be > 0 and <= 1")
|
|
return fraction
|
|
|
|
|
|
def parse_job_cpu_fraction(argv):
|
|
fraction = DEFAULT_JOB_CPU_FRACTION
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--job-cpu-fraction="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
fraction = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --job-cpu-fraction value: {value}") from exc
|
|
if fraction <= 0 or fraction > 1:
|
|
raise SystemExit("--job-cpu-fraction must be > 0 and <= 1")
|
|
return fraction
|
|
|
|
|
|
def parse_runner_mem_swap_factor(argv):
|
|
factor = DEFAULT_RUNNER_MEM_SWAP_FACTOR
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--runner-memory-swap-factor="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
factor = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --runner-memory-swap-factor value: {value}") from exc
|
|
if factor < 1:
|
|
raise SystemExit("--runner-memory-swap-factor must be >= 1")
|
|
return factor
|
|
|
|
|
|
def parse_job_mem_swap_factor(argv):
|
|
factor = DEFAULT_JOB_MEM_SWAP_FACTOR
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--job-memory-swap-factor="):
|
|
_, value = arg.split("=", 1)
|
|
try:
|
|
factor = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --job-memory-swap-factor value: {value}") from exc
|
|
if factor < 1:
|
|
raise SystemExit("--job-memory-swap-factor must be >= 1")
|
|
return factor
|
|
|
|
|
|
def parse_runner_pids_limit(argv):
|
|
value = DEFAULT_RUNNER_PIDS_LIMIT
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--runner-pids-limit="):
|
|
_, raw = arg.split("=", 1)
|
|
try:
|
|
value = int(raw)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --runner-pids-limit value: {raw}") from exc
|
|
if value < 64:
|
|
raise SystemExit("--runner-pids-limit must be >= 64")
|
|
return value
|
|
|
|
|
|
def parse_job_pids_limit(argv):
|
|
value = DEFAULT_JOB_PIDS_LIMIT
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--job-pids-limit="):
|
|
_, raw = arg.split("=", 1)
|
|
try:
|
|
value = int(raw)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --job-pids-limit value: {raw}") from exc
|
|
if value < 32:
|
|
raise SystemExit("--job-pids-limit must be >= 32")
|
|
return value
|
|
|
|
|
|
def parse_job_nofile_soft(argv):
|
|
value = DEFAULT_JOB_NOFILE_SOFT
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--job-ulimit-nofile-soft="):
|
|
_, raw = arg.split("=", 1)
|
|
try:
|
|
value = int(raw)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --job-ulimit-nofile-soft value: {raw}") from exc
|
|
if value < 256:
|
|
raise SystemExit("--job-ulimit-nofile-soft must be >= 256")
|
|
return value
|
|
|
|
|
|
def parse_job_nofile_hard(argv):
|
|
value = DEFAULT_JOB_NOFILE_HARD
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--job-ulimit-nofile-hard="):
|
|
_, raw = arg.split("=", 1)
|
|
try:
|
|
value = int(raw)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --job-ulimit-nofile-hard value: {raw}") from exc
|
|
if value < 256:
|
|
raise SystemExit("--job-ulimit-nofile-hard must be >= 256")
|
|
return value
|
|
|
|
|
|
def parse_runner_max_parallel_jobs(argv):
|
|
value = DEFAULT_RUNNER_MAX_PARALLEL_JOBS
|
|
for arg in argv[1:]:
|
|
if arg.startswith("--runner-max-parallel-jobs="):
|
|
_, raw = arg.split("=", 1)
|
|
try:
|
|
value = int(raw)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid --runner-max-parallel-jobs value: {raw}") from exc
|
|
if value < 1:
|
|
raise SystemExit("--runner-max-parallel-jobs must be >= 1")
|
|
return value
|
|
|
|
|
|
def parse_dry_run(argv):
|
|
return "--dry-run" in argv[1:]
|
|
|
|
def ssh(host, cmd):
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-o",
|
|
"ConnectTimeout=30",
|
|
"-o",
|
|
"ServerAliveInterval=30",
|
|
"-o",
|
|
"ServerAliveCountMax=6",
|
|
host,
|
|
cmd,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return proc.returncode, proc.stdout + proc.stderr
|
|
|
|
|
|
def scp(local_path, host, remote_path):
|
|
proc = subprocess.run(
|
|
[
|
|
"scp",
|
|
"-o",
|
|
"ConnectTimeout=30",
|
|
"-o",
|
|
"ServerAliveInterval=30",
|
|
"-o",
|
|
"ServerAliveCountMax=6",
|
|
local_path,
|
|
f"{host}:{remote_path}",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return proc.returncode, proc.stdout + proc.stderr
|
|
|
|
|
|
def service_block_bounds(lines, start_index):
|
|
service_indent = len(lines[start_index]) - len(lines[start_index].lstrip(" "))
|
|
block_start = start_index + 1
|
|
block_end = len(lines)
|
|
j = block_start
|
|
while j < len(lines):
|
|
cur = lines[j]
|
|
cur_stripped = cur.strip()
|
|
if cur_stripped:
|
|
cur_indent = len(cur) - len(cur.lstrip(" "))
|
|
if cur_indent <= service_indent:
|
|
block_end = j
|
|
break
|
|
j += 1
|
|
return service_indent, block_start, block_end
|
|
|
|
|
|
def upsert_service_key(compose_text, service_names, key, value):
|
|
lines = compose_text.splitlines()
|
|
i = 0
|
|
|
|
while i < len(lines):
|
|
stripped = lines[i].strip()
|
|
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
|
|
i += 1
|
|
continue
|
|
|
|
service_indent, block_start, block_end = service_block_bounds(lines, i)
|
|
field_indent = " " * (service_indent + 2)
|
|
|
|
key_prefix = f"{key}:"
|
|
found_idx = None
|
|
for k in range(block_start, block_end):
|
|
if lines[k].strip().startswith(key_prefix):
|
|
found_idx = k
|
|
break
|
|
|
|
new_line = f"{field_indent}{key}: {value}"
|
|
if found_idx is not None:
|
|
lines[found_idx] = new_line
|
|
i = block_end
|
|
continue
|
|
|
|
lines.insert(block_start, new_line)
|
|
i = block_end + 1
|
|
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def remove_service_key(compose_text, service_names, key):
|
|
lines = compose_text.splitlines()
|
|
i = 0
|
|
|
|
while i < len(lines):
|
|
stripped = lines[i].strip()
|
|
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
|
|
i += 1
|
|
continue
|
|
|
|
_, block_start, block_end = service_block_bounds(lines, i)
|
|
key_prefix = f"{key}:"
|
|
filtered = []
|
|
for idx in range(block_start, block_end):
|
|
if not lines[idx].strip().startswith(key_prefix):
|
|
filtered.append(lines[idx])
|
|
|
|
lines = lines[:block_start] + filtered + lines[block_end:]
|
|
i = block_start + len(filtered)
|
|
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def upsert_service_deploy_limits(compose_text, service_names, memory_value, cpu_value, pids_value):
|
|
lines = compose_text.splitlines()
|
|
i = 0
|
|
|
|
while i < len(lines):
|
|
stripped = lines[i].strip()
|
|
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
|
|
i += 1
|
|
continue
|
|
|
|
service_indent, block_start, service_block_end = service_block_bounds(lines, i)
|
|
deploy_indent = " " * (service_indent + 2)
|
|
resources_indent = " " * (service_indent + 4)
|
|
limits_indent = " " * (service_indent + 6)
|
|
value_indent = " " * (service_indent + 8)
|
|
deploy_prefix = f"{deploy_indent}deploy:"
|
|
|
|
# Remove an existing deploy block under this service so we can rewrite it canonically.
|
|
deploy_idx = None
|
|
deploy_end = None
|
|
for idx in range(block_start, service_block_end):
|
|
if lines[idx].startswith(deploy_prefix):
|
|
deploy_idx = idx
|
|
j = idx + 1
|
|
while j < service_block_end:
|
|
cur = lines[j]
|
|
cur_stripped = cur.strip()
|
|
if cur_stripped:
|
|
cur_indent = len(cur) - len(cur.lstrip(" "))
|
|
if cur_indent <= service_indent + 2:
|
|
break
|
|
j += 1
|
|
deploy_end = j
|
|
break
|
|
|
|
if deploy_idx is not None and deploy_end is not None:
|
|
lines = lines[:deploy_idx] + lines[deploy_end:]
|
|
service_block_end -= deploy_end - deploy_idx
|
|
|
|
insert_lines = [
|
|
f"{deploy_indent}deploy:",
|
|
f"{resources_indent}resources:",
|
|
f"{limits_indent}limits:",
|
|
f"{value_indent}memory: {memory_value}",
|
|
f"{value_indent}cpus: '{cpu_value}'",
|
|
f"{value_indent}pids: {pids_value}",
|
|
]
|
|
lines = lines[:block_start] + insert_lines + lines[block_start:]
|
|
i = service_block_end + len(insert_lines)
|
|
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def upsert_compose_env_var(compose_text, var_name, var_value, anchor_line):
|
|
lines = compose_text.splitlines()
|
|
new_line = f" - {var_name}={var_value}"
|
|
replaced = False
|
|
|
|
for idx, line in enumerate(lines):
|
|
stripped = line.strip()
|
|
if stripped.startswith(f"- {var_name}="):
|
|
lines[idx] = new_line
|
|
replaced = True
|
|
|
|
if not replaced:
|
|
for idx, line in enumerate(lines):
|
|
if line.strip() == anchor_line.strip():
|
|
lines.insert(idx + 1, new_line)
|
|
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def parse_mem_value_to_mib(raw_value):
|
|
value = raw_value.strip().lower()
|
|
if not value:
|
|
return None
|
|
|
|
# Compose mem_limit values are typically 4096m / 4g; inspect values are bytes.
|
|
units = {
|
|
"k": 1.0 / 1024.0,
|
|
"m": 1.0,
|
|
"g": 1024.0,
|
|
}
|
|
for suffix, factor in units.items():
|
|
if value.endswith(suffix):
|
|
num = value[: -len(suffix)]
|
|
try:
|
|
return int(float(num) * factor)
|
|
except ValueError:
|
|
return None
|
|
|
|
if value.isdigit():
|
|
as_int = int(value)
|
|
# Docker inspect HostConfig.Memory reports bytes.
|
|
if as_int > 0:
|
|
return int(as_int / (1024 * 1024))
|
|
return None
|
|
|
|
|
|
def find_service_key_value(compose_text, service_names, key):
|
|
lines = compose_text.splitlines()
|
|
i = 0
|
|
|
|
while i < len(lines):
|
|
stripped = lines[i].strip()
|
|
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
|
|
i += 1
|
|
continue
|
|
|
|
service_indent = len(lines[i]) - len(lines[i].lstrip(" "))
|
|
j = i + 1
|
|
while j < len(lines):
|
|
cur = lines[j]
|
|
cur_stripped = cur.strip()
|
|
if cur_stripped:
|
|
cur_indent = len(cur) - len(cur.lstrip(" "))
|
|
if cur_indent <= service_indent:
|
|
break
|
|
if cur_stripped.startswith(f"{key}:"):
|
|
return cur_stripped.split(":", 1)[1].strip()
|
|
j += 1
|
|
i = j
|
|
|
|
return None
|
|
|
|
|
|
def find_service_memory_value(compose_text, service_names):
|
|
lines = compose_text.splitlines()
|
|
i = 0
|
|
|
|
while i < len(lines):
|
|
stripped = lines[i].strip()
|
|
if stripped.rstrip(":") not in service_names or not stripped.endswith(":"):
|
|
i += 1
|
|
continue
|
|
|
|
_, block_start, block_end = service_block_bounds(lines, i)
|
|
for idx in range(block_start, block_end):
|
|
cur = lines[idx].strip()
|
|
if cur.startswith("memory:"):
|
|
return cur.split(":", 1)[1].strip()
|
|
i = block_end
|
|
|
|
return None
|
|
|
|
|
|
def build_compose(
|
|
compose_text,
|
|
runner_mem_limit,
|
|
runner_memswap_limit,
|
|
runner_cpu_limit,
|
|
runner_pids_limit,
|
|
runner_max_parallel_jobs,
|
|
job_mem_limit,
|
|
job_memswap_limit,
|
|
job_cpu_limit,
|
|
job_pids_limit,
|
|
job_nofile_soft,
|
|
job_nofile_hard,
|
|
):
|
|
updated = compose_text
|
|
|
|
volume_line = " - ./daemon.json:/etc/docker/daemon.json:ro"
|
|
add_volume = " - ./config.yaml:/config.yaml:ro"
|
|
if add_volume not in updated and volume_line in updated:
|
|
updated = updated.replace(volume_line, volume_line + "\n" + add_volume)
|
|
|
|
force_pull_line = " - GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL=${GITEA_RUNNER_JOB_CONTAINER_FORCE_PULL:-false}"
|
|
add_cfg_env = " - CONFIG_FILE=/config.yaml"
|
|
if force_pull_line in updated:
|
|
updated = upsert_compose_env_var(updated, "CONFIG_FILE", "/config.yaml", force_pull_line)
|
|
updated = upsert_compose_env_var(
|
|
updated,
|
|
"GITEA_RUNNER_MAX_PARALLEL_JOBS",
|
|
str(runner_max_parallel_jobs),
|
|
force_pull_line,
|
|
)
|
|
updated = upsert_compose_env_var(
|
|
updated,
|
|
"GITEA_RUNNER_JOB_CONTAINER_OPTIONS",
|
|
(
|
|
f"--memory={job_mem_limit} --memory-swap={job_memswap_limit} "
|
|
f"--cpus={job_cpu_limit} --pids-limit={job_pids_limit} "
|
|
f"--ulimit=nofile={job_nofile_soft}:{job_nofile_hard}"
|
|
),
|
|
force_pull_line,
|
|
)
|
|
elif add_cfg_env not in updated:
|
|
updated = updated.replace(force_pull_line, force_pull_line + "\n" + add_cfg_env)
|
|
|
|
runner_services = {"act_runner_1", "act_runner_2", "runner1", "runner2"}
|
|
updated = remove_service_key(updated, runner_services, "mem_limit")
|
|
updated = remove_service_key(updated, runner_services, "pids_limit")
|
|
updated = remove_service_key(updated, runner_services, "pids")
|
|
updated = upsert_service_key(updated, runner_services, "memswap_limit", runner_memswap_limit)
|
|
updated = upsert_service_deploy_limits(
|
|
updated,
|
|
runner_services,
|
|
runner_mem_limit,
|
|
runner_cpu_limit,
|
|
str(runner_pids_limit),
|
|
)
|
|
|
|
return updated
|
|
|
|
|
|
def host_total_mem_mib(host):
|
|
rc, mem_kib = ssh(host, "awk '/MemTotal:/ {print $2}' /proc/meminfo")
|
|
if rc != 0 or not mem_kib.strip().isdigit():
|
|
raise RuntimeError(f"Could not read MemTotal on {host}")
|
|
return int(int(mem_kib.strip()) / 1024.0)
|
|
|
|
|
|
def host_total_cpus(host):
|
|
rc, cpu_count = ssh(host, "nproc")
|
|
if rc != 0:
|
|
raise RuntimeError(f"Could not read CPU count on {host}")
|
|
value = cpu_count.strip()
|
|
try:
|
|
cpus = float(value)
|
|
except ValueError as exc:
|
|
raise RuntimeError(f"Invalid CPU count on {host}: {value}") from exc
|
|
if cpus <= 0:
|
|
raise RuntimeError(f"Non-positive CPU count on {host}: {value}")
|
|
return cpus
|
|
|
|
|
|
def host_mem_limit(host, host_mem_share, fraction):
|
|
total_mib = host_total_mem_mib(host)
|
|
host_share_mib = int(total_mib * host_mem_share)
|
|
target_mib = int(host_share_mib * fraction)
|
|
if target_mib < MIN_RUNNER_MEM_MIB:
|
|
target_mib = MIN_RUNNER_MEM_MIB
|
|
return (
|
|
f"{target_mib}m",
|
|
target_mib,
|
|
f"host-total({total_mib}MiB) * host-share({host_mem_share:.2f}) -> {host_share_mib}MiB * mem-fraction({fraction:.2f})",
|
|
)
|
|
|
|
|
|
def host_cpu_limit(host, host_cpu_share, fraction):
|
|
total_cpus = host_total_cpus(host)
|
|
host_share_cpus = total_cpus * host_cpu_share
|
|
target_cpus = host_share_cpus * fraction
|
|
if target_cpus <= 0.01:
|
|
target_cpus = 0.01
|
|
return (
|
|
f"{target_cpus:.2f}",
|
|
target_cpus,
|
|
f"host-total({total_cpus:.0f}cpu) * host-share({host_cpu_share:.2f}) -> {host_share_cpus:.2f}cpu * cpu-fraction({fraction:.2f})",
|
|
)
|
|
|
|
|
|
mem_fraction = parse_mem_fraction(sys.argv)
|
|
job_mem_fraction = parse_job_mem_fraction(sys.argv)
|
|
host_mem_share = parse_host_mem_share(sys.argv)
|
|
host_cpu_share = parse_host_cpu_share(sys.argv)
|
|
cpu_fraction = parse_cpu_fraction(sys.argv)
|
|
job_cpu_fraction = parse_job_cpu_fraction(sys.argv)
|
|
runner_mem_swap_factor = parse_runner_mem_swap_factor(sys.argv)
|
|
job_mem_swap_factor = parse_job_mem_swap_factor(sys.argv)
|
|
runner_pids_limit = parse_runner_pids_limit(sys.argv)
|
|
job_pids_limit = parse_job_pids_limit(sys.argv)
|
|
job_nofile_soft = parse_job_nofile_soft(sys.argv)
|
|
job_nofile_hard = parse_job_nofile_hard(sys.argv)
|
|
runner_max_parallel_jobs = parse_runner_max_parallel_jobs(sys.argv)
|
|
dry_run = parse_dry_run(sys.argv)
|
|
|
|
if job_nofile_hard < job_nofile_soft:
|
|
raise SystemExit("--job-ulimit-nofile-hard must be >= --job-ulimit-nofile-soft")
|
|
|
|
print(f"Host mem share: {host_mem_share:.2f}")
|
|
print(f"Runner mem fraction: {mem_fraction:.2f}")
|
|
print(f"Job mem fraction: {job_mem_fraction:.2f}")
|
|
print(f"Host cpu share: {host_cpu_share:.2f}")
|
|
print(f"Runner cpu fraction: {cpu_fraction:.2f}")
|
|
print(f"Job cpu fraction: {job_cpu_fraction:.2f}")
|
|
print(f"Runner memory swap factor: {runner_mem_swap_factor:.2f}")
|
|
print(f"Job memory swap factor: {job_mem_swap_factor:.2f}")
|
|
print(f"Runner pids limit: {runner_pids_limit}")
|
|
print(f"Job pids limit: {job_pids_limit}")
|
|
print(f"Job nofile soft/hard: {job_nofile_soft}/{job_nofile_hard}")
|
|
print(f"Runner max parallel jobs: {runner_max_parallel_jobs}")
|
|
print(f"Dry run: {dry_run}")
|
|
print("Fetching canonical runner config template")
|
|
rc, template = ssh(TEMPLATE_HOST, f"cat {TEMPLATE_PATH}")
|
|
if rc != 0 or not template.strip():
|
|
print("Failed to fetch template from kankali")
|
|
raise SystemExit(2)
|
|
|
|
now = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
local_cfg = os.path.join(tmpdir, "config.yaml")
|
|
with open(local_cfg, "w", encoding="utf-8") as f:
|
|
f.write(template)
|
|
|
|
for host, root in HOSTS.items():
|
|
print(f"\n===== {host} =====")
|
|
|
|
rc, compose = ssh(host, f"cat {root}/compose.yml")
|
|
if rc != 0:
|
|
print("Could not read compose.yml")
|
|
continue
|
|
|
|
try:
|
|
runner_mem_limit, runner_mem_mib, baseline_source = host_mem_limit(
|
|
host,
|
|
host_mem_share,
|
|
mem_fraction,
|
|
)
|
|
runner_cpu_limit, runner_cpu_value, cpu_source = host_cpu_limit(
|
|
host,
|
|
host_cpu_share,
|
|
cpu_fraction,
|
|
)
|
|
except RuntimeError as exc:
|
|
print(str(exc))
|
|
continue
|
|
|
|
print(f"runner_mem_baseline={baseline_source}")
|
|
print(f"runner_mem_limit={runner_mem_limit} ({runner_mem_mib} MiB)")
|
|
runner_memswap_mib = int(runner_mem_mib * runner_mem_swap_factor)
|
|
if runner_memswap_mib < runner_mem_mib:
|
|
runner_memswap_mib = runner_mem_mib
|
|
runner_memswap_limit = f"{runner_memswap_mib}m"
|
|
print(f"runner_memswap_limit={runner_memswap_limit} ({runner_memswap_mib} MiB)")
|
|
print(f"runner_cpu_baseline={cpu_source}")
|
|
print(f"runner_cpu_limit={runner_cpu_limit} cpu")
|
|
|
|
job_mem_mib = int(runner_mem_mib * job_mem_fraction)
|
|
if job_mem_mib < MIN_JOB_MEM_MIB:
|
|
job_mem_mib = MIN_JOB_MEM_MIB
|
|
if job_mem_mib >= runner_mem_mib and runner_mem_mib > 1:
|
|
job_mem_mib = runner_mem_mib - 1
|
|
job_mem_limit = f"{job_mem_mib}m"
|
|
print(f"job_mem_limit={job_mem_limit} ({job_mem_mib} MiB)")
|
|
|
|
job_memswap_mib = int(job_mem_mib * job_mem_swap_factor)
|
|
if job_memswap_mib < job_mem_mib:
|
|
job_memswap_mib = job_mem_mib
|
|
job_memswap_limit = f"{job_memswap_mib}m"
|
|
print(f"job_memswap_limit={job_memswap_limit} ({job_memswap_mib} MiB)")
|
|
|
|
job_cpu_value = runner_cpu_value * job_cpu_fraction
|
|
if job_cpu_value <= 0.01:
|
|
job_cpu_value = 0.01
|
|
if job_cpu_value >= runner_cpu_value and runner_cpu_value > 0.02:
|
|
job_cpu_value = runner_cpu_value - 0.01
|
|
job_cpu_limit = f"{job_cpu_value:.2f}"
|
|
print(f"job_cpu_limit={job_cpu_limit} cpu")
|
|
print(f"job_pids_limit={job_pids_limit}")
|
|
print(f"job_nofile_soft/hard={job_nofile_soft}/{job_nofile_hard}")
|
|
|
|
new_compose = build_compose(
|
|
compose,
|
|
runner_mem_limit,
|
|
runner_memswap_limit,
|
|
runner_cpu_limit,
|
|
runner_pids_limit,
|
|
runner_max_parallel_jobs,
|
|
job_mem_limit,
|
|
job_memswap_limit,
|
|
job_cpu_limit,
|
|
job_pids_limit,
|
|
job_nofile_soft,
|
|
job_nofile_hard,
|
|
)
|
|
|
|
if dry_run:
|
|
print("dry-run=true; skipping upload and restart")
|
|
continue
|
|
|
|
rc, _ = ssh(host, f"mkdir -p {root}/backups")
|
|
rc, _ = ssh(host, f"cp -f {root}/compose.yml {root}/backups/compose.yml.{now}")
|
|
rc, _ = ssh(host, f"test -f {root}/config.yaml")
|
|
if rc == 0:
|
|
ssh(host, f"cp -f {root}/config.yaml {root}/backups/config.yaml.{now}")
|
|
|
|
local_compose = os.path.join(tmpdir, f"compose.{host}.yml")
|
|
with open(local_compose, "w", encoding="utf-8") as f:
|
|
f.write(new_compose)
|
|
|
|
rc, out = scp(local_compose, host, f"{root}/compose.yml")
|
|
if rc != 0:
|
|
print("Failed to upload compose.yml")
|
|
print(out)
|
|
continue
|
|
|
|
rc, out = scp(local_cfg, host, f"{root}/config.yaml")
|
|
if rc != 0:
|
|
print("Failed to upload config.yaml")
|
|
print(out)
|
|
continue
|
|
|
|
rc, out = ssh(host, f"grep -n 'config.yaml:/config.yaml:ro' {root}/compose.yml")
|
|
print(out.strip())
|
|
rc, out = ssh(host, f"grep -n 'CONFIG_FILE=/config.yaml' {root}/compose.yml")
|
|
print(out.strip())
|
|
rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_JOB_CONTAINER_OPTIONS=' {root}/compose.yml")
|
|
print(out.strip())
|
|
rc, out = ssh(host, f"grep -n 'GITEA_RUNNER_MAX_PARALLEL_JOBS=' {root}/compose.yml")
|
|
print(out.strip())
|
|
rc, out = ssh(host, f"grep -nE 'memory:|cpus:|pids:|memswap_limit:|pids_limit:' {root}/compose.yml")
|
|
print(out.strip())
|
|
|
|
rc, out = ssh(host, f"docker compose -f {root}/compose.yml up -d act_runner_1")
|
|
if rc != 0:
|
|
print(out.strip())
|
|
print("Failed to apply compose changes on host")
|
|
continue
|
|
print(out.strip())
|
|
|
|
rc, out = ssh(host, "docker inspect gitea-act-runner-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'")
|
|
print("mounts:", out.strip())
|
|
|
|
print("Done")
|
|
|
|
print("\nDeployment complete")
|