diff --git a/.gitea/workflows/cicd-checks.yaml b/.gitea/workflows/cicd-checks.yaml index feac8ea..c473dde 100644 --- a/.gitea/workflows/cicd-checks.yaml +++ b/.gitea/workflows/cicd-checks.yaml @@ -26,6 +26,7 @@ jobs: setup: name: Setup Checks Context runs-on: ubuntu-act + timeout-minutes: 8 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} steps: @@ -59,6 +60,7 @@ jobs: run-check: name: ${{ matrix.name }} runs-on: ubuntu-act + timeout-minutes: 20 needs: setup strategy: fail-fast: false @@ -110,7 +112,24 @@ jobs: echo "Using cached CICD image: ${IMAGE}" else echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - docker pull "${IMAGE}" + pulled=false + for i in 1 2 3; do + echo "Pull attempt ${i}/3 for ${IMAGE}" + if docker pull "${IMAGE}"; then + pulled=true + break + fi + if [ "${i}" -lt 3 ]; then + sleep_seconds=$((5 * i)) + echo "Pull failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + + if [ "${pulled}" != "true" ]; then + echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}" + exit 1 + fi fi - name: Run pre-commit hook @@ -126,6 +145,7 @@ jobs: dispatch-tests: name: Dispatch CICD Tests runs-on: ubuntu-act + timeout-minutes: 15 needs: [setup, run-check] steps: - name: Dispatch tests workflow @@ -151,7 +171,6 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1" REPO_OWNER="${REPO_FULL%/*}" REPO_NAME="${REPO_FULL#*/}" TARGET_REF="${HEAD_REF:-${REF_NAME}}" @@ -159,39 +178,64 @@ jobs: echo "trace_id=${TRACE_ID}" echo "target_ref=${TARGET_REF}" - echo "dispatch_api_base=${API_BASE}" - payload=$(cat </dev/null 2>&1; then + return 0 + fi + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + command -v curl >/dev/null 2>&1 } - EOF - ) - DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/cicd-tests.yaml/dispatches" - RESPONSE_FILE="/tmp/dispatch-response.txt" - : > "${RESPONSE_FILE}" - dispatch_start_ms=$(date +%s%3N) - HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "Content-Type: application/json" \ - "${DISPATCH_URL}" \ - -d "${payload}" || true) - dispatch_end_ms=$(date +%s%3N) - dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms)) + ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; } - echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}" - echo "dispatch_http_code=${HTTP_CODE}" + HELPER_PATH="/tmp/dispatch-workflow.sh" - if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then - echo "❌ Dispatch failed with HTTP ${HTTP_CODE}" - echo "Dispatch URL: ${DISPATCH_URL}" - echo "Response body:" - cat "${RESPONSE_FILE}" || true + fetch_dispatch_helper() { + local helper_ref="$1" + local api_base + for api_base in "${CANDIDATE_API_BASES[@]}"; do + helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" + if curl -fsS --connect-timeout 5 --max-time 20 \ + -H "Authorization: token ${DISPATCH_TOKEN}" \ + -H "User-Agent: plex-playlist-cicd-checks" \ + -o "${HELPER_PATH}" \ + "${helper_url}"; then + chmod +x "${HELPER_PATH}" + return 0 + fi + done + return 1 + } + + if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then + echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" exit 1 fi + + DISPATCH_ARGS=( + --token "${DISPATCH_TOKEN}" + --repo "${REPO_FULL}" + --workflow "cicd-tests.yaml" + --ref "${TARGET_REF}" + --head-sha "${HEAD_SHA}" + --source-workflow "CICD Checks" + --trace-id "${TRACE_ID}" + ) + + for API_BASE in "${CANDIDATE_API_BASES[@]}"; do + DISPATCH_ARGS+=(--api-base "${API_BASE}") + done + + "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" diff --git a/.gitea/workflows/cicd-start.yaml b/.gitea/workflows/cicd-start.yaml index 25207a6..45bfd49 100644 --- a/.gitea/workflows/cicd-start.yaml +++ b/.gitea/workflows/cicd-start.yaml @@ -28,6 +28,7 @@ jobs: name: Sanity and Base Decision # Keep startup lightweight to avoid runner image setup stalls. runs-on: ubuntu-latest + timeout-minutes: 12 outputs: base_needed: ${{ steps.base-decision.outputs.base_needed }} head_sha: ${{ steps.base-decision.outputs.head_sha }} @@ -112,20 +113,6 @@ jobs: TARGET_WORKFLOW="docker-build-main.yaml" fi - payload=$(cat < "${RESPONSE_FILE}" + fetch_dispatch_helper() { + local helper_ref="$1" + local api_base + for api_base in "${CANDIDATE_API_BASES[@]}"; do + helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" + if curl -fsS --connect-timeout 5 --max-time 20 \ + -H "Authorization: token ${DISPATCH_TOKEN}" \ + -H "User-Agent: plex-playlist-cicd-start" \ + -o "${HELPER_PATH}" \ + "${helper_url}"; then + chmod +x "${HELPER_PATH}" + return 0 + fi + done + return 1 + } - dispatch_start_ms=$(date +%s%3N) - HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "Content-Type: application/json" \ - -H "User-Agent: plex-playlist-cicd-start" \ - "${DISPATCH_URL}" \ - -d "${payload}" || true) - dispatch_end_ms=$(date +%s%3N) - dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms)) - echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}" - echo "dispatch_http_code=${HTTP_CODE}" - - if [[ "${HTTP_CODE}" =~ ^[0-9]+$ ]] && [ "${HTTP_CODE}" -ge 200 ] && [ "${HTTP_CODE}" -lt 300 ]; then - echo "✓ Dispatch succeeded via ${API_BASE}" - dispatched=true - break - fi - - echo "Dispatch attempt failed via ${API_BASE} (HTTP ${HTTP_CODE})" - if [ -s "${RESPONSE_FILE}" ]; then - echo "Response body:" - cat "${RESPONSE_FILE}" - fi - done - - if [ "${dispatched}" != "true" ]; then - echo "❌ Dispatch failed on all candidate API endpoints" + if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then + echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" exit 1 fi + + DISPATCH_ARGS=( + --token "${DISPATCH_TOKEN}" + --repo "${REPO_FULL}" + --workflow "${TARGET_WORKFLOW}" + --ref "${TARGET_REF}" + --head-sha "${HEAD_SHA}" + --source-workflow "CICD Start" + --trace-id "${TRACE_ID}" + --base-needed "${BASE_NEEDED}" + --base-hash "${BASE_HASH}" + ) + + for API_BASE in "${CANDIDATE_API_BASES[@]}"; do + DISPATCH_ARGS+=(--api-base "${API_BASE}") + done + + "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" diff --git a/.gitea/workflows/cicd-tests.yaml b/.gitea/workflows/cicd-tests.yaml index 813e7e1..5fe217d 100644 --- a/.gitea/workflows/cicd-tests.yaml +++ b/.gitea/workflows/cicd-tests.yaml @@ -26,6 +26,7 @@ jobs: setup: name: Setup Tests Context runs-on: ubuntu-act + timeout-minutes: 8 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} steps: @@ -59,6 +60,7 @@ jobs: backend-tests: name: Backend Tests runs-on: ubuntu-act + timeout-minutes: 25 needs: setup steps: - &configure_registry_host_step @@ -78,7 +80,24 @@ jobs: echo "Using cached CICD image: ${IMAGE}" else echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin - docker pull "${IMAGE}" + pulled=false + for i in 1 2 3; do + echo "Pull attempt ${i}/3 for ${IMAGE}" + if docker pull "${IMAGE}"; then + pulled=true + break + fi + if [ "${i}" -lt 3 ]; then + sleep_seconds=$((5 * i)) + echo "Pull failed; retrying in ${sleep_seconds}s" + sleep "${sleep_seconds}" + fi + done + + if [ "${pulled}" != "true" ]; then + echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}" + exit 1 + fi fi - name: Run backend tests with coverage @@ -94,6 +113,7 @@ jobs: frontend-tests: name: Frontend Tests runs-on: ubuntu-act + timeout-minutes: 25 needs: setup steps: - *configure_registry_host_step @@ -110,6 +130,7 @@ jobs: xdoctest: name: Backend Doctests runs-on: ubuntu-act + timeout-minutes: 15 needs: setup steps: - *configure_registry_host_step @@ -127,6 +148,7 @@ jobs: integration-tests: name: Integration Tests runs-on: ubuntu-act + timeout-minutes: 20 needs: [setup, backend-tests] steps: - *configure_registry_host_step @@ -148,6 +170,7 @@ jobs: e2e-tests: name: End-to-End Tests runs-on: ubuntu-act + timeout-minutes: 30 needs: [setup, frontend-tests] steps: - *configure_registry_host_step diff --git a/.gitea/workflows/cicd.yml b/.gitea/workflows/cicd.yml index cb7a015..d1893e4 100644 --- a/.gitea/workflows/cicd.yml +++ b/.gitea/workflows/cicd.yml @@ -16,6 +16,7 @@ jobs: name: Launch CICD Start # Lightweight dispatch-only job; avoid dependency on ubuntu-act image startup. runs-on: ubuntu-latest + timeout-minutes: 8 steps: - name: Configure registry host resolution run: | @@ -45,7 +46,6 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1" REPO_OWNER="${REPO_FULL%/*}" REPO_NAME="${REPO_FULL#*/}" TARGET_REF="${HEAD_REF:-${REF_NAME}}" @@ -53,41 +53,63 @@ jobs: echo "trace_id=${TRACE_ID}" echo "target_ref=${TARGET_REF}" - echo "api_base=${API_BASE}" + CANDIDATE_API_BASES=() + if [ -n "${GITHUB_SERVER_URL:-}" ]; then + CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") + fi + CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") + CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") - payload=$(cat </dev/null 2>&1; then + return 0 + fi + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + command -v curl >/dev/null 2>&1 } - EOF - ) - DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/cicd-start.yaml/dispatches" - RESPONSE_FILE="/tmp/dispatch-response.txt" - : > "${RESPONSE_FILE}" - dispatch_start_ms=$(date +%s%3N) - HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "Content-Type: application/json" \ - "${DISPATCH_URL}" \ - -d "${payload}" || true) - dispatch_end_ms=$(date +%s%3N) - dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms)) + ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; } - echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}" - echo "dispatch_http_code=${HTTP_CODE}" + HELPER_PATH="/tmp/dispatch-workflow.sh" - if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then - echo "❌ Dispatch failed with HTTP ${HTTP_CODE}" - echo "Dispatch URL: ${DISPATCH_URL}" - echo "Response body:" - cat "${RESPONSE_FILE}" || true + fetch_dispatch_helper() { + local helper_ref="$1" + local api_base + for api_base in "${CANDIDATE_API_BASES[@]}"; do + helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" + if curl -fsS --connect-timeout 5 --max-time 20 \ + -H "Authorization: token ${DISPATCH_TOKEN}" \ + -H "User-Agent: plex-playlist-cicd" \ + -o "${HELPER_PATH}" \ + "${helper_url}"; then + chmod +x "${HELPER_PATH}" + return 0 + fi + done + return 1 + } + + if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then + echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" exit 1 fi - echo "✓ CICD Start dispatch succeeded" + DISPATCH_ARGS=( + --token "${DISPATCH_TOKEN}" + --repo "${REPO_FULL}" + --workflow "cicd-start.yaml" + --ref "${TARGET_REF}" + --head-sha "${HEAD_SHA}" + --source-workflow "CICD" + --trace-id "${TRACE_ID}" + ) + + for API_BASE in "${CANDIDATE_API_BASES[@]}"; do + DISPATCH_ARGS+=(--api-base "${API_BASE}") + done + + "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" diff --git a/.gitea/workflows/docker-build-base.yaml b/.gitea/workflows/docker-build-base.yaml index 1dad47a..618e7c4 100644 --- a/.gitea/workflows/docker-build-base.yaml +++ b/.gitea/workflows/docker-build-base.yaml @@ -331,7 +331,6 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1" REPO_OWNER="${REPO_FULL%/*}" REPO_NAME="${REPO_FULL#*/}" TARGET_REF="${HEAD_REF:-${REF_NAME}}" @@ -339,7 +338,13 @@ jobs: echo "trace_id=${TRACE_ID}" echo "target_ref=${TARGET_REF}" - echo "dispatch_api_base=${API_BASE}" + + CANDIDATE_API_BASES=() + if [ -n "${GITHUB_SERVER_URL:-}" ]; then + CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") + fi + CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") + CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") if ! command -v curl >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive @@ -347,39 +352,44 @@ jobs: apt-get install -y -qq curl ca-certificates fi - payload=$(cat < "${RESPONSE_FILE}" - dispatch_start_ms=$(date +%s%3N) - HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "Content-Type: application/json" \ - "${DISPATCH_URL}" \ - -d "${payload}" || true) - dispatch_end_ms=$(date +%s%3N) - dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms)) - - echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}" - echo "dispatch_http_code=${HTTP_CODE}" - - if ! [[ "${HTTP_CODE}" =~ ^[0-9]+$ ]] || [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then - echo "❌ Dispatch failed with HTTP ${HTTP_CODE}" - echo "Dispatch URL: ${DISPATCH_URL}" - echo "Response body:" - cat "${RESPONSE_FILE}" || true + if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then + echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" exit 1 fi + + DISPATCH_ARGS=( + --token "${DISPATCH_TOKEN}" + --repo "${REPO_FULL}" + --workflow "docker-build-main.yaml" + --ref "${TARGET_REF}" + --head-sha "${HEAD_SHA}" + --source-workflow "CICD Base Image" + --trace-id "${TRACE_ID}" + --base-needed "true" + --base-hash "${BASE_HASH}" + ) + + for API_BASE in "${CANDIDATE_API_BASES[@]}"; do + DISPATCH_ARGS+=(--api-base "${API_BASE}") + done + + "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" diff --git a/.gitea/workflows/docker-build-main.yaml b/.gitea/workflows/docker-build-main.yaml index 1ca3aa9..385ee96 100644 --- a/.gitea/workflows/docker-build-main.yaml +++ b/.gitea/workflows/docker-build-main.yaml @@ -36,6 +36,7 @@ jobs: name: Build and Push CICD Complete Image # Heavy Docker build: schedule only on 8GB Pi runners. runs-on: ubuntu-act-8gb + timeout-minutes: 60 outputs: head_sha: ${{ steps.meta.outputs.head_sha }} @@ -183,6 +184,7 @@ jobs: dispatch-checks: name: Dispatch CICD Checks runs-on: ubuntu-act + timeout-minutes: 15 needs: build if: needs.build.result == 'success' steps: @@ -209,7 +211,6 @@ jobs: echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts fi - API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1" REPO_OWNER="${REPO_FULL%/*}" REPO_NAME="${REPO_FULL#*/}" TARGET_REF="${HEAD_REF:-${REF_NAME}}" @@ -217,39 +218,64 @@ jobs: echo "trace_id=${TRACE_ID}" echo "target_ref=${TARGET_REF}" - echo "dispatch_api_base=${API_BASE}" - payload=$(cat </dev/null 2>&1; then + return 0 + fi + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + command -v curl >/dev/null 2>&1 } - EOF - ) - DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/cicd-checks.yaml/dispatches" - RESPONSE_FILE="/tmp/dispatch-response.txt" - : > "${RESPONSE_FILE}" - dispatch_start_ms=$(date +%s%3N) - HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \ - -H "Authorization: token ${DISPATCH_TOKEN}" \ - -H "Content-Type: application/json" \ - "${DISPATCH_URL}" \ - -d "${payload}" || true) - dispatch_end_ms=$(date +%s%3N) - dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms)) + ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; } - echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}" - echo "dispatch_http_code=${HTTP_CODE}" + HELPER_PATH="/tmp/dispatch-workflow.sh" - if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then - echo "❌ Dispatch failed with HTTP ${HTTP_CODE}" - echo "Dispatch URL: ${DISPATCH_URL}" - echo "Response body:" - cat "${RESPONSE_FILE}" || true + fetch_dispatch_helper() { + local helper_ref="$1" + local api_base + for api_base in "${CANDIDATE_API_BASES[@]}"; do + helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}" + if curl -fsS --connect-timeout 5 --max-time 20 \ + -H "Authorization: token ${DISPATCH_TOKEN}" \ + -H "User-Agent: plex-playlist-cicd-main" \ + -o "${HELPER_PATH}" \ + "${helper_url}"; then + chmod +x "${HELPER_PATH}" + return 0 + fi + done + return 1 + } + + if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then + echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository" exit 1 fi + + DISPATCH_ARGS=( + --token "${DISPATCH_TOKEN}" + --repo "${REPO_FULL}" + --workflow "cicd-checks.yaml" + --ref "${TARGET_REF}" + --head-sha "${HEAD_SHA}" + --source-workflow "CICD Main Build" + --trace-id "${TRACE_ID}" + ) + + for API_BASE in "${CANDIDATE_API_BASES[@]}"; do + DISPATCH_ARGS+=(--api-base "${API_BASE}") + done + + "${HELPER_PATH}" "${DISPATCH_ARGS[@]}" diff --git a/.gitea/workflows/renovate.yml b/.gitea/workflows/renovate.yml index 1776fa7..be50f77 100644 --- a/.gitea/workflows/renovate.yml +++ b/.gitea/workflows/renovate.yml @@ -17,6 +17,7 @@ jobs: name: Renovate Dependencies # Non-heavy workflow: allow any host exposing the generic ubuntu-act label. runs-on: ubuntu-act + timeout-minutes: 90 steps: - name: Setup Node.js for Renovate diff --git a/.gitea/workflows/runner-canary.yaml b/.gitea/workflows/runner-canary.yaml index 748ee08..21f873d 100644 --- a/.gitea/workflows/runner-canary.yaml +++ b/.gitea/workflows/runner-canary.yaml @@ -10,6 +10,7 @@ jobs: name: Canary (${{ matrix.label }}) if: github.event_name == 'schedule' runs-on: ${{ matrix.label }} + timeout-minutes: 6 strategy: fail-fast: false matrix: @@ -48,6 +49,7 @@ jobs: name: Canary Heavy (${{ matrix.label }}) if: github.event_name == 'workflow_dispatch' runs-on: ${{ matrix.label }} + timeout-minutes: 8 strategy: fail-fast: false matrix: diff --git a/scripts/dispatch-workflow.sh b/scripts/dispatch-workflow.sh index 8949138..92fc669 100755 --- a/scripts/dispatch-workflow.sh +++ b/scripts/dispatch-workflow.sh @@ -1,13 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -if [[ -n "${GITHUB_SERVER_URL:-}" ]]; then - API_BASE="${GITHUB_SERVER_URL%/}/api/v1" -elif [[ -n "${GITEA_SERVER_URL:-}" ]]; then - API_BASE="${GITEA_SERVER_URL%/}/api/v1" -else - API_BASE="http://kankali.darkhelm.lan:3000/api/v1" -fi TOKEN="" REPO_FULL="" WORKFLOW_FILE="" @@ -15,11 +8,84 @@ TARGET_REF="" HEAD_SHA="" SOURCE_WORKFLOW="" BASE_NEEDED="" +BASE_HASH="" +TRACE_ID="" +MAX_ROUNDS=3 + +declare -a API_BASES=() +declare -a EXTRA_INPUTS=() + +usage() { + cat <<'EOF' +Usage: + dispatch-workflow.sh \ + --token \ + --repo \ + --workflow \ + --ref \ + --head-sha \ + --source-workflow \ + [--api-base ]... \ + [--base-needed ] \ + [--base-hash ] \ + [--trace-id ] \ + [--input ]... \ + [--max-rounds ] +EOF +} + +ensure_curl() { + if command -v curl >/dev/null 2>&1; then + return 0 + fi + + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq curl ca-certificates + fi + + if command -v curl >/dev/null 2>&1; then + return 0 + fi + + echo "curl is required for dispatch and could not be installed" >&2 + return 1 +} + +build_default_api_bases() { + if [[ -n "${GITHUB_SERVER_URL:-}" ]]; then + API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1") + fi + if [[ -n "${GITEA_SERVER_URL:-}" ]]; then + API_BASES+=("${GITEA_SERVER_URL%/}/api/v1") + fi + if [[ -n "${GITEA_SSH_HOST:-}" ]]; then + API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1") + fi + if [[ -n "${GITEA_REGISTRY_HOST:-}" ]]; then + API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1") + fi + if [[ -n "${GITEA_REGISTRY_IP:-}" ]]; then + API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1") + fi + if [[ ${#API_BASES[@]} -eq 0 ]]; then + API_BASES+=("http://kankali.darkhelm.lan:3001/api/v1") + fi +} + +append_input_field() { + local key="$1" + local value="$2" + [[ -z "$value" ]] && return 0 + PAYLOAD_INPUTS+=$',\n' + PAYLOAD_INPUTS+=" \"${key}\": \"${value}\"" +} while [[ $# -gt 0 ]]; do case "$1" in --api-base) - API_BASE="$2" + API_BASES+=("$2") shift 2 ;; --token) @@ -50,8 +116,29 @@ while [[ $# -gt 0 ]]; do BASE_NEEDED="$2" shift 2 ;; + --base-hash) + BASE_HASH="$2" + shift 2 + ;; + --trace-id) + TRACE_ID="$2" + shift 2 + ;; + --input) + EXTRA_INPUTS+=("$2") + shift 2 + ;; + --max-rounds) + MAX_ROUNDS="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; *) echo "Unknown argument: $1" >&2 + usage exit 2 ;; esac @@ -59,40 +146,102 @@ done if [[ -z "$TOKEN" || -z "$REPO_FULL" || -z "$WORKFLOW_FILE" || -z "$TARGET_REF" || -z "$HEAD_SHA" || -z "$SOURCE_WORKFLOW" ]]; then echo "Missing required arguments" >&2 + usage + exit 2 +fi + +if ! [[ "$MAX_ROUNDS" =~ ^[0-9]+$ ]] || [[ "$MAX_ROUNDS" -lt 1 ]]; then + echo "--max-rounds must be a positive integer" >&2 exit 2 fi REPO_OWNER="${REPO_FULL%/*}" REPO_NAME="${REPO_FULL#*/}" -if [[ -n "$BASE_NEEDED" ]]; then - PAYLOAD=$(cat <&2 + exit 2 + fi + key="${kv%%=*}" + value="${kv#*=}" + append_input_field "$key" "$value" +done + +PAYLOAD=$(cat < "${response_file}" + + echo "dispatch_attempt=${attempt}/${total_attempts}" + echo "dispatch_round=${round}/${MAX_ROUNDS}" + echo "dispatch_api_base=${api_base}" + + dispatch_start_ms=$(date +%s%3N) + http_code=$(curl -sS --connect-timeout 5 --max-time 20 -o "${response_file}" -w "%{http_code}" -X POST \ + -H "Authorization: token ${TOKEN}" \ + -H "Content-Type: application/json" \ + "${dispatch_url}" \ + -d "${PAYLOAD}" || true) + dispatch_end_ms=$(date +%s%3N) + dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms)) + + echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}" + echo "dispatch_http_code=${http_code}" + + if [[ "${http_code}" =~ ^[0-9]+$ ]] && [[ "${http_code}" -ge 200 ]] && [[ "${http_code}" -lt 300 ]]; then + echo "Dispatch succeeded" + dispatched=true + break 2 + fi + + echo "Dispatch attempt failed via ${api_base} (HTTP ${http_code})" + if [[ -s "${response_file}" ]]; then + echo "Response body:" + cat "${response_file}" || true + fi + done + + sleep_seconds=$((2 ** round)) + echo "Dispatch round ${round} failed; backing off for ${sleep_seconds}s" + sleep "${sleep_seconds}" +done + +if [[ "${dispatched}" != "true" ]]; then + echo "Dispatch failed on all endpoints after ${MAX_ROUNDS} rounds" >&2 + exit 1 +fi