More CICD fixes again again again again again again again again again again again again again again again again again

Signed-off-by: copilotcoder <copilotcoder@darkhelm.org>

.gitea/workflows/cicd-checks.yaml

@@ -26,6 +26,7 @@ jobs:
   setup:
     name: Setup Checks Context
     runs-on: ubuntu-act
+    timeout-minutes: 8
     outputs:
       head_sha: ${{ steps.meta.outputs.head_sha }}
     steps:
@@ -59,6 +60,7 @@ jobs:
   run-check:
     name: ${{ matrix.name }}
     runs-on: ubuntu-act
+    timeout-minutes: 20
     needs: setup
     strategy:
       fail-fast: false
@@ -110,7 +112,24 @@ jobs:
             echo "Using cached CICD image: ${IMAGE}"
           else
             echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
-            docker pull "${IMAGE}"
+            pulled=false
+            for i in 1 2 3; do
+              echo "Pull attempt ${i}/3 for ${IMAGE}"
+              if docker pull "${IMAGE}"; then
+                pulled=true
+                break
+              fi
+              if [ "${i}" -lt 3 ]; then
+                sleep_seconds=$((5 * i))
+                echo "Pull failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+            done
+
+            if [ "${pulled}" != "true" ]; then
+              echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
+              exit 1
+            fi
           fi
 
       - name: Run pre-commit hook
@@ -126,6 +145,7 @@ jobs:
   dispatch-tests:
     name: Dispatch CICD Tests
     runs-on: ubuntu-act
+    timeout-minutes: 15
     needs: [setup, run-check]
     steps:
       - name: Dispatch tests workflow
@@ -151,7 +171,6 @@ jobs:
             echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
           fi
 
-          API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1"
           REPO_OWNER="${REPO_FULL%/*}"
           REPO_NAME="${REPO_FULL#*/}"
           TARGET_REF="${HEAD_REF:-${REF_NAME}}"
@@ -159,39 +178,64 @@ jobs:
 
           echo "trace_id=${TRACE_ID}"
           echo "target_ref=${TARGET_REF}"
-          echo "dispatch_api_base=${API_BASE}"
 
-          payload=$(cat <<EOF
-          {
-            "ref": "${TARGET_REF}",
-            "inputs": {
-              "head_sha": "${HEAD_SHA}",
-              "source_workflow": "CICD Checks",
-              "trace_id": "${TRACE_ID}"
-            }
+          CANDIDATE_API_BASES=()
+          if [ -n "${GITHUB_SERVER_URL:-}" ]; then
+            CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
+          fi
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
+
+          ensure_curl() {
+            if command -v curl >/dev/null 2>&1; then
+              return 0
+            fi
+            if command -v apt-get >/dev/null 2>&1; then
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update -qq
+              apt-get install -y -qq curl ca-certificates
+            fi
+            command -v curl >/dev/null 2>&1
           }
-          EOF
-          )
 
-          DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/cicd-tests.yaml/dispatches"
-          RESPONSE_FILE="/tmp/dispatch-response.txt"
-          : > "${RESPONSE_FILE}"
-          dispatch_start_ms=$(date +%s%3N)
-          HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \
-            -H "Authorization: token ${DISPATCH_TOKEN}" \
-            -H "Content-Type: application/json" \
-            "${DISPATCH_URL}" \
-            -d "${payload}" || true)
-          dispatch_end_ms=$(date +%s%3N)
-          dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms))
+          ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; }
 
-          echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}"
-          echo "dispatch_http_code=${HTTP_CODE}"
+          HELPER_PATH="/tmp/dispatch-workflow.sh"
 
-          if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then
-            echo "❌ Dispatch failed with HTTP ${HTTP_CODE}"
-            echo "Dispatch URL: ${DISPATCH_URL}"
-            echo "Response body:"
-            cat "${RESPONSE_FILE}" || true
+          fetch_dispatch_helper() {
+            local helper_ref="$1"
+            local api_base
+            for api_base in "${CANDIDATE_API_BASES[@]}"; do
+              helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
+              if curl -fsS --connect-timeout 5 --max-time 20 \
+                -H "Authorization: token ${DISPATCH_TOKEN}" \
+                -H "User-Agent: plex-playlist-cicd-checks" \
+                -o "${HELPER_PATH}" \
+                "${helper_url}"; then
+                chmod +x "${HELPER_PATH}"
+                return 0
+              fi
+            done
+            return 1
+          }
+
+          if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
+            echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
             exit 1
           fi
+
+          DISPATCH_ARGS=(
+            --token "${DISPATCH_TOKEN}"
+            --repo "${REPO_FULL}"
+            --workflow "cicd-tests.yaml"
+            --ref "${TARGET_REF}"
+            --head-sha "${HEAD_SHA}"
+            --source-workflow "CICD Checks"
+            --trace-id "${TRACE_ID}"
+          )
+
+          for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
+            DISPATCH_ARGS+=(--api-base "${API_BASE}")
+          done
+
+          "${HELPER_PATH}" "${DISPATCH_ARGS[@]}"

.gitea/workflows/cicd-start.yaml

@@ -28,6 +28,7 @@ jobs:
     name: Sanity and Base Decision
     # Keep startup lightweight to avoid runner image setup stalls.
     runs-on: ubuntu-latest
+    timeout-minutes: 12
     outputs:
       base_needed: ${{ steps.base-decision.outputs.base_needed }}
       head_sha: ${{ steps.base-decision.outputs.head_sha }}
@@ -112,20 +113,6 @@ jobs:
             TARGET_WORKFLOW="docker-build-main.yaml"
           fi
 
-          payload=$(cat <<EOF
-          {
-            "ref": "${TARGET_REF}",
-            "inputs": {
-              "head_sha": "${HEAD_SHA}",
-              "base_hash": "${BASE_HASH}",
-              "source_workflow": "CICD Start",
-              "base_needed": "${BASE_NEEDED}",
-              "trace_id": "${TRACE_ID}"
-            }
-          }
-          EOF
-          )
-
           CANDIDATE_API_BASES=()
           if [ -n "${GITHUB_SERVER_URL:-}" ]; then
             CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
@@ -161,44 +148,44 @@ jobs:
 
           ensure_curl
 
-          dispatched=false
-          total_attempts=${#CANDIDATE_API_BASES[@]}
-          attempt=0
-          for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
-            attempt=$((attempt + 1))
-            DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/${TARGET_WORKFLOW}/dispatches"
-            echo "dispatch_attempt=${attempt}/${total_attempts}"
-            echo "dispatch_api_base=${API_BASE}"
+          HELPER_PATH="/tmp/dispatch-workflow.sh"
 
-            RESPONSE_FILE="/tmp/dispatch-response.txt"
-            : > "${RESPONSE_FILE}"
+          fetch_dispatch_helper() {
+            local helper_ref="$1"
+            local api_base
+            for api_base in "${CANDIDATE_API_BASES[@]}"; do
+              helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
+              if curl -fsS --connect-timeout 5 --max-time 20 \
+                -H "Authorization: token ${DISPATCH_TOKEN}" \
+                -H "User-Agent: plex-playlist-cicd-start" \
+                -o "${HELPER_PATH}" \
+                "${helper_url}"; then
+                chmod +x "${HELPER_PATH}"
+                return 0
+              fi
+            done
+            return 1
+          }
 
-            dispatch_start_ms=$(date +%s%3N)
-            HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \
-              -H "Authorization: token ${DISPATCH_TOKEN}" \
-              -H "Content-Type: application/json" \
-              -H "User-Agent: plex-playlist-cicd-start" \
-              "${DISPATCH_URL}" \
-              -d "${payload}" || true)
-            dispatch_end_ms=$(date +%s%3N)
-            dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms))
-            echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}"
-            echo "dispatch_http_code=${HTTP_CODE}"
-
-            if [[ "${HTTP_CODE}" =~ ^[0-9]+$ ]] && [ "${HTTP_CODE}" -ge 200 ] && [ "${HTTP_CODE}" -lt 300 ]; then
-              echo "✓ Dispatch succeeded via ${API_BASE}"
-              dispatched=true
-              break
-            fi
-
-            echo "Dispatch attempt failed via ${API_BASE} (HTTP ${HTTP_CODE})"
-            if [ -s "${RESPONSE_FILE}" ]; then
-              echo "Response body:"
-              cat "${RESPONSE_FILE}"
-            fi
-          done
-
-          if [ "${dispatched}" != "true" ]; then
-            echo "❌ Dispatch failed on all candidate API endpoints"
+          if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
+            echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
             exit 1
           fi
+
+          DISPATCH_ARGS=(
+            --token "${DISPATCH_TOKEN}"
+            --repo "${REPO_FULL}"
+            --workflow "${TARGET_WORKFLOW}"
+            --ref "${TARGET_REF}"
+            --head-sha "${HEAD_SHA}"
+            --source-workflow "CICD Start"
+            --trace-id "${TRACE_ID}"
+            --base-needed "${BASE_NEEDED}"
+            --base-hash "${BASE_HASH}"
+          )
+
+          for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
+            DISPATCH_ARGS+=(--api-base "${API_BASE}")
+          done
+
+          "${HELPER_PATH}" "${DISPATCH_ARGS[@]}"

.gitea/workflows/cicd-tests.yaml

@@ -26,6 +26,7 @@ jobs:
   setup:
     name: Setup Tests Context
     runs-on: ubuntu-act
+    timeout-minutes: 8
     outputs:
       head_sha: ${{ steps.meta.outputs.head_sha }}
     steps:
@@ -59,6 +60,7 @@ jobs:
   backend-tests:
     name: Backend Tests
     runs-on: ubuntu-act
+    timeout-minutes: 25
     needs: setup
     steps:
       - &configure_registry_host_step
@@ -78,7 +80,24 @@ jobs:
             echo "Using cached CICD image: ${IMAGE}"
           else
             echo "${{ secrets.PACKAGE_ACCESS_TOKEN }}" | docker login "http://${GITEA_REGISTRY}" -u "${{ github.actor }}" --password-stdin
-            docker pull "${IMAGE}"
+            pulled=false
+            for i in 1 2 3; do
+              echo "Pull attempt ${i}/3 for ${IMAGE}"
+              if docker pull "${IMAGE}"; then
+                pulled=true
+                break
+              fi
+              if [ "${i}" -lt 3 ]; then
+                sleep_seconds=$((5 * i))
+                echo "Pull failed; retrying in ${sleep_seconds}s"
+                sleep "${sleep_seconds}"
+              fi
+            done
+
+            if [ "${pulled}" != "true" ]; then
+              echo "❌ Failed to pull CICD image after 3 attempts: ${IMAGE}"
+              exit 1
+            fi
           fi
 
       - name: Run backend tests with coverage
@@ -94,6 +113,7 @@ jobs:
   frontend-tests:
     name: Frontend Tests
     runs-on: ubuntu-act
+    timeout-minutes: 25
     needs: setup
     steps:
       - *configure_registry_host_step
@@ -110,6 +130,7 @@ jobs:
   xdoctest:
     name: Backend Doctests
     runs-on: ubuntu-act
+    timeout-minutes: 15
     needs: setup
     steps:
       - *configure_registry_host_step
@@ -127,6 +148,7 @@ jobs:
   integration-tests:
     name: Integration Tests
     runs-on: ubuntu-act
+    timeout-minutes: 20
     needs: [setup, backend-tests]
     steps:
       - *configure_registry_host_step
@@ -148,6 +170,7 @@ jobs:
   e2e-tests:
     name: End-to-End Tests
     runs-on: ubuntu-act
+    timeout-minutes: 30
     needs: [setup, frontend-tests]
     steps:
       - *configure_registry_host_step

.gitea/workflows/cicd.yml

@@ -16,6 +16,7 @@ jobs:
     name: Launch CICD Start
     # Lightweight dispatch-only job; avoid dependency on ubuntu-act image startup.
     runs-on: ubuntu-latest
+    timeout-minutes: 8
     steps:
       - name: Configure registry host resolution
         run: |
@@ -45,7 +46,6 @@ jobs:
             echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
           fi
 
-          API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1"
           REPO_OWNER="${REPO_FULL%/*}"
           REPO_NAME="${REPO_FULL#*/}"
           TARGET_REF="${HEAD_REF:-${REF_NAME}}"
@@ -53,41 +53,63 @@ jobs:
 
           echo "trace_id=${TRACE_ID}"
           echo "target_ref=${TARGET_REF}"
-          echo "api_base=${API_BASE}"
+          CANDIDATE_API_BASES=()
+          if [ -n "${GITHUB_SERVER_URL:-}" ]; then
+            CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
+          fi
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
 
-          payload=$(cat <<EOF
-          {
-            "ref": "${TARGET_REF}",
-            "inputs": {
-              "head_sha": "${HEAD_SHA}",
-              "source_workflow": "CICD",
-              "trace_id": "${TRACE_ID}"
-            }
+          ensure_curl() {
+            if command -v curl >/dev/null 2>&1; then
+              return 0
+            fi
+            if command -v apt-get >/dev/null 2>&1; then
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update -qq
+              apt-get install -y -qq curl ca-certificates
+            fi
+            command -v curl >/dev/null 2>&1
           }
-          EOF
-          )
 
-          DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/cicd-start.yaml/dispatches"
-          RESPONSE_FILE="/tmp/dispatch-response.txt"
-          : > "${RESPONSE_FILE}"
-          dispatch_start_ms=$(date +%s%3N)
-          HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \
-            -H "Authorization: token ${DISPATCH_TOKEN}" \
-            -H "Content-Type: application/json" \
-            "${DISPATCH_URL}" \
-            -d "${payload}" || true)
-          dispatch_end_ms=$(date +%s%3N)
-          dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms))
+          ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; }
 
-          echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}"
-          echo "dispatch_http_code=${HTTP_CODE}"
+          HELPER_PATH="/tmp/dispatch-workflow.sh"
 
-          if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then
-            echo "❌ Dispatch failed with HTTP ${HTTP_CODE}"
-            echo "Dispatch URL: ${DISPATCH_URL}"
-            echo "Response body:"
-            cat "${RESPONSE_FILE}" || true
+          fetch_dispatch_helper() {
+            local helper_ref="$1"
+            local api_base
+            for api_base in "${CANDIDATE_API_BASES[@]}"; do
+              helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
+              if curl -fsS --connect-timeout 5 --max-time 20 \
+                -H "Authorization: token ${DISPATCH_TOKEN}" \
+                -H "User-Agent: plex-playlist-cicd" \
+                -o "${HELPER_PATH}" \
+                "${helper_url}"; then
+                chmod +x "${HELPER_PATH}"
+                return 0
+              fi
+            done
+            return 1
+          }
+
+          if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
+            echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
             exit 1
           fi
 
-          echo "✓ CICD Start dispatch succeeded"
+          DISPATCH_ARGS=(
+            --token "${DISPATCH_TOKEN}"
+            --repo "${REPO_FULL}"
+            --workflow "cicd-start.yaml"
+            --ref "${TARGET_REF}"
+            --head-sha "${HEAD_SHA}"
+            --source-workflow "CICD"
+            --trace-id "${TRACE_ID}"
+          )
+
+          for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
+            DISPATCH_ARGS+=(--api-base "${API_BASE}")
+          done
+
+          "${HELPER_PATH}" "${DISPATCH_ARGS[@]}"

.gitea/workflows/docker-build-base.yaml

@@ -331,7 +331,6 @@ jobs:
             echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
           fi
 
-          API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1"
           REPO_OWNER="${REPO_FULL%/*}"
           REPO_NAME="${REPO_FULL#*/}"
           TARGET_REF="${HEAD_REF:-${REF_NAME}}"
@@ -339,7 +338,13 @@ jobs:
 
           echo "trace_id=${TRACE_ID}"
           echo "target_ref=${TARGET_REF}"
-          echo "dispatch_api_base=${API_BASE}"
+
+          CANDIDATE_API_BASES=()
+          if [ -n "${GITHUB_SERVER_URL:-}" ]; then
+            CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
+          fi
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
 
           if ! command -v curl >/dev/null 2>&1; then
             export DEBIAN_FRONTEND=noninteractive
@@ -347,39 +352,44 @@ jobs:
             apt-get install -y -qq curl ca-certificates
           fi
 
-          payload=$(cat <<EOF
-          {
-            "ref": "${TARGET_REF}",
-            "inputs": {
-              "head_sha": "${HEAD_SHA}",
-              "base_hash": "${BASE_HASH}",
-              "source_workflow": "CICD Base Image",
-              "base_needed": "true",
-              "trace_id": "${TRACE_ID}"
-            }
+          HELPER_PATH="/tmp/dispatch-workflow.sh"
+
+          fetch_dispatch_helper() {
+            local helper_ref="$1"
+            local api_base
+            for api_base in "${CANDIDATE_API_BASES[@]}"; do
+              helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
+              if curl -fsS --connect-timeout 5 --max-time 20 \
+                -H "Authorization: token ${DISPATCH_TOKEN}" \
+                -H "User-Agent: plex-playlist-cicd-base" \
+                -o "${HELPER_PATH}" \
+                "${helper_url}"; then
+                chmod +x "${HELPER_PATH}"
+                return 0
+              fi
+            done
+            return 1
           }
-          EOF
-          )
 
-          DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/docker-build-main.yaml/dispatches"
-          RESPONSE_FILE="/tmp/dispatch-response.txt"
-          : > "${RESPONSE_FILE}"
-          dispatch_start_ms=$(date +%s%3N)
-          HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \
-            -H "Authorization: token ${DISPATCH_TOKEN}" \
-            -H "Content-Type: application/json" \
-            "${DISPATCH_URL}" \
-            -d "${payload}" || true)
-          dispatch_end_ms=$(date +%s%3N)
-          dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms))
-
-          echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}"
-          echo "dispatch_http_code=${HTTP_CODE}"
-
-          if ! [[ "${HTTP_CODE}" =~ ^[0-9]+$ ]] || [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then
-            echo "❌ Dispatch failed with HTTP ${HTTP_CODE}"
-            echo "Dispatch URL: ${DISPATCH_URL}"
-            echo "Response body:"
-            cat "${RESPONSE_FILE}" || true
+          if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
+            echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
             exit 1
           fi
+
+          DISPATCH_ARGS=(
+            --token "${DISPATCH_TOKEN}"
+            --repo "${REPO_FULL}"
+            --workflow "docker-build-main.yaml"
+            --ref "${TARGET_REF}"
+            --head-sha "${HEAD_SHA}"
+            --source-workflow "CICD Base Image"
+            --trace-id "${TRACE_ID}"
+            --base-needed "true"
+            --base-hash "${BASE_HASH}"
+          )
+
+          for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
+            DISPATCH_ARGS+=(--api-base "${API_BASE}")
+          done
+
+          "${HELPER_PATH}" "${DISPATCH_ARGS[@]}"

.gitea/workflows/docker-build-main.yaml

@@ -36,6 +36,7 @@ jobs:
     name: Build and Push CICD Complete Image
     # Heavy Docker build: schedule only on 8GB Pi runners.
     runs-on: ubuntu-act-8gb
+    timeout-minutes: 60
     outputs:
       head_sha: ${{ steps.meta.outputs.head_sha }}
 
@@ -183,6 +184,7 @@ jobs:
   dispatch-checks:
     name: Dispatch CICD Checks
     runs-on: ubuntu-act
+    timeout-minutes: 15
     needs: build
     if: needs.build.result == 'success'
     steps:
@@ -209,7 +211,6 @@ jobs:
             echo "${GITEA_REGISTRY_IP} ${GITEA_REGISTRY_HOST}" >> /etc/hosts
           fi
 
-          API_BASE="http://${GITEA_REGISTRY_IP}:3001/api/v1"
           REPO_OWNER="${REPO_FULL%/*}"
           REPO_NAME="${REPO_FULL#*/}"
           TARGET_REF="${HEAD_REF:-${REF_NAME}}"
@@ -217,39 +218,64 @@ jobs:
 
           echo "trace_id=${TRACE_ID}"
           echo "target_ref=${TARGET_REF}"
-          echo "dispatch_api_base=${API_BASE}"
 
-          payload=$(cat <<EOF
-          {
-            "ref": "${TARGET_REF}",
-            "inputs": {
-              "head_sha": "${HEAD_SHA}",
-              "source_workflow": "CICD Main Build",
-              "trace_id": "${TRACE_ID}"
-            }
+          CANDIDATE_API_BASES=()
+          if [ -n "${GITHUB_SERVER_URL:-}" ]; then
+            CANDIDATE_API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
+          fi
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
+          CANDIDATE_API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
+
+          ensure_curl() {
+            if command -v curl >/dev/null 2>&1; then
+              return 0
+            fi
+            if command -v apt-get >/dev/null 2>&1; then
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update -qq
+              apt-get install -y -qq curl ca-certificates
+            fi
+            command -v curl >/dev/null 2>&1
           }
-          EOF
-          )
 
-          DISPATCH_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/cicd-checks.yaml/dispatches"
-          RESPONSE_FILE="/tmp/dispatch-response.txt"
-          : > "${RESPONSE_FILE}"
-          dispatch_start_ms=$(date +%s%3N)
-          HTTP_CODE=$(curl -sS --connect-timeout 5 --max-time 20 -o "${RESPONSE_FILE}" -w "%{http_code}" -X POST \
-            -H "Authorization: token ${DISPATCH_TOKEN}" \
-            -H "Content-Type: application/json" \
-            "${DISPATCH_URL}" \
-            -d "${payload}" || true)
-          dispatch_end_ms=$(date +%s%3N)
-          dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms))
+          ensure_curl || { echo "❌ curl unavailable for dispatch"; exit 1; }
 
-          echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}"
-          echo "dispatch_http_code=${HTTP_CODE}"
+          HELPER_PATH="/tmp/dispatch-workflow.sh"
 
-          if [ "${HTTP_CODE}" -lt 200 ] || [ "${HTTP_CODE}" -ge 300 ]; then
-            echo "❌ Dispatch failed with HTTP ${HTTP_CODE}"
-            echo "Dispatch URL: ${DISPATCH_URL}"
-            echo "Response body:"
-            cat "${RESPONSE_FILE}" || true
+          fetch_dispatch_helper() {
+            local helper_ref="$1"
+            local api_base
+            for api_base in "${CANDIDATE_API_BASES[@]}"; do
+              helper_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/raw/scripts/dispatch-workflow.sh?ref=${helper_ref}"
+              if curl -fsS --connect-timeout 5 --max-time 20 \
+                -H "Authorization: token ${DISPATCH_TOKEN}" \
+                -H "User-Agent: plex-playlist-cicd-main" \
+                -o "${HELPER_PATH}" \
+                "${helper_url}"; then
+                chmod +x "${HELPER_PATH}"
+                return 0
+              fi
+            done
+            return 1
+          }
+
+          if ! fetch_dispatch_helper "${TARGET_REF}" && ! fetch_dispatch_helper "${HEAD_SHA}"; then
+            echo "❌ Failed to fetch scripts/dispatch-workflow.sh from repository"
             exit 1
           fi
+
+          DISPATCH_ARGS=(
+            --token "${DISPATCH_TOKEN}"
+            --repo "${REPO_FULL}"
+            --workflow "cicd-checks.yaml"
+            --ref "${TARGET_REF}"
+            --head-sha "${HEAD_SHA}"
+            --source-workflow "CICD Main Build"
+            --trace-id "${TRACE_ID}"
+          )
+
+          for API_BASE in "${CANDIDATE_API_BASES[@]}"; do
+            DISPATCH_ARGS+=(--api-base "${API_BASE}")
+          done
+
+          "${HELPER_PATH}" "${DISPATCH_ARGS[@]}"

.gitea/workflows/renovate.yml

@@ -17,6 +17,7 @@ jobs:
     name: Renovate Dependencies
     # Non-heavy workflow: allow any host exposing the generic ubuntu-act label.
     runs-on: ubuntu-act
+    timeout-minutes: 90
 
     steps:
       - name: Setup Node.js for Renovate

.gitea/workflows/runner-canary.yaml

@@ -10,6 +10,7 @@ jobs:
     name: Canary (${{ matrix.label }})
     if: github.event_name == 'schedule'
     runs-on: ${{ matrix.label }}
+    timeout-minutes: 6
     strategy:
       fail-fast: false
       matrix:
@@ -48,6 +49,7 @@ jobs:
     name: Canary Heavy (${{ matrix.label }})
     if: github.event_name == 'workflow_dispatch'
     runs-on: ${{ matrix.label }}
+    timeout-minutes: 8
     strategy:
       fail-fast: false
       matrix:

scripts/dispatch-workflow.sh

@@ -1,13 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [[ -n "${GITHUB_SERVER_URL:-}" ]]; then
-  API_BASE="${GITHUB_SERVER_URL%/}/api/v1"
-elif [[ -n "${GITEA_SERVER_URL:-}" ]]; then
-  API_BASE="${GITEA_SERVER_URL%/}/api/v1"
-else
-  API_BASE="http://kankali.darkhelm.lan:3000/api/v1"
-fi
 TOKEN=""
 REPO_FULL=""
 WORKFLOW_FILE=""
@@ -15,11 +8,84 @@ TARGET_REF=""
 HEAD_SHA=""
 SOURCE_WORKFLOW=""
 BASE_NEEDED=""
+BASE_HASH=""
+TRACE_ID=""
+MAX_ROUNDS=3
+
+declare -a API_BASES=()
+declare -a EXTRA_INPUTS=()
+
+usage() {
+  cat <<'EOF'
+Usage:
+  dispatch-workflow.sh \
+    --token <token> \
+    --repo <owner/repo> \
+    --workflow <workflow-file> \
+    --ref <branch-or-ref> \
+    --head-sha <sha> \
+    --source-workflow <name> \
+    [--api-base <base-url>]... \
+    [--base-needed <true|false>] \
+    [--base-hash <hash>] \
+    [--trace-id <id>] \
+    [--input <key=value>]... \
+    [--max-rounds <n>]
+EOF
+}
+
+ensure_curl() {
+  if command -v curl >/dev/null 2>&1; then
+    return 0
+  fi
+
+  if command -v apt-get >/dev/null 2>&1; then
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update -qq
+    apt-get install -y -qq curl ca-certificates
+  fi
+
+  if command -v curl >/dev/null 2>&1; then
+    return 0
+  fi
+
+  echo "curl is required for dispatch and could not be installed" >&2
+  return 1
+}
+
+build_default_api_bases() {
+  if [[ -n "${GITHUB_SERVER_URL:-}" ]]; then
+    API_BASES+=("${GITHUB_SERVER_URL%/}/api/v1")
+  fi
+  if [[ -n "${GITEA_SERVER_URL:-}" ]]; then
+    API_BASES+=("${GITEA_SERVER_URL%/}/api/v1")
+  fi
+  if [[ -n "${GITEA_SSH_HOST:-}" ]]; then
+    API_BASES+=("http://${GITEA_SSH_HOST}:3001/api/v1")
+  fi
+  if [[ -n "${GITEA_REGISTRY_HOST:-}" ]]; then
+    API_BASES+=("http://${GITEA_REGISTRY_HOST}:3001/api/v1")
+  fi
+  if [[ -n "${GITEA_REGISTRY_IP:-}" ]]; then
+    API_BASES+=("http://${GITEA_REGISTRY_IP}:3001/api/v1")
+  fi
+  if [[ ${#API_BASES[@]} -eq 0 ]]; then
+    API_BASES+=("http://kankali.darkhelm.lan:3001/api/v1")
+  fi
+}
+
+append_input_field() {
+  local key="$1"
+  local value="$2"
+  [[ -z "$value" ]] && return 0
+  PAYLOAD_INPUTS+=$',\n'
+  PAYLOAD_INPUTS+="    \"${key}\": \"${value}\""
+}
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --api-base)
-      API_BASE="$2"
+      API_BASES+=("$2")
       shift 2
       ;;
     --token)
@@ -50,8 +116,29 @@ while [[ $# -gt 0 ]]; do
       BASE_NEEDED="$2"
       shift 2
       ;;
+    --base-hash)
+      BASE_HASH="$2"
+      shift 2
+      ;;
+    --trace-id)
+      TRACE_ID="$2"
+      shift 2
+      ;;
+    --input)
+      EXTRA_INPUTS+=("$2")
+      shift 2
+      ;;
+    --max-rounds)
+      MAX_ROUNDS="$2"
+      shift 2
+      ;;
+    --help|-h)
+      usage
+      exit 0
+      ;;
     *)
       echo "Unknown argument: $1" >&2
+      usage
       exit 2
       ;;
   esac
@@ -59,40 +146,102 @@ done
 
 if [[ -z "$TOKEN" || -z "$REPO_FULL" || -z "$WORKFLOW_FILE" || -z "$TARGET_REF" || -z "$HEAD_SHA" || -z "$SOURCE_WORKFLOW" ]]; then
   echo "Missing required arguments" >&2
+  usage
+  exit 2
+fi
+
+if ! [[ "$MAX_ROUNDS" =~ ^[0-9]+$ ]] || [[ "$MAX_ROUNDS" -lt 1 ]]; then
+  echo "--max-rounds must be a positive integer" >&2
   exit 2
 fi
 
 REPO_OWNER="${REPO_FULL%/*}"
 REPO_NAME="${REPO_FULL#*/}"
 
-if [[ -n "$BASE_NEEDED" ]]; then
-  PAYLOAD=$(cat <<EOF
-{
-  "ref": "${TARGET_REF}",
-  "inputs": {
-    "head_sha": "${HEAD_SHA}",
-    "source_workflow": "${SOURCE_WORKFLOW}",
-    "base_needed": "${BASE_NEEDED}"
-  }
-}
-EOF
-)
-else
-  PAYLOAD=$(cat <<EOF
-{
-  "ref": "${TARGET_REF}",
-  "inputs": {
-    "head_sha": "${HEAD_SHA}",
-    "source_workflow": "${SOURCE_WORKFLOW}"
-  }
-}
-EOF
-)
+if [[ ${#API_BASES[@]} -eq 0 ]]; then
+  build_default_api_bases
 fi
 
+if [[ -z "$TRACE_ID" ]]; then
+  TRACE_ID="dispatch-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${HEAD_SHA:0:8}"
+fi
+
+PAYLOAD_INPUTS=$'    "head_sha": "'"${HEAD_SHA}"$'",\n    "source_workflow": "'"${SOURCE_WORKFLOW}"$'"'
+append_input_field "trace_id" "$TRACE_ID"
+append_input_field "base_needed" "$BASE_NEEDED"
+append_input_field "base_hash" "$BASE_HASH"
+
+for kv in "${EXTRA_INPUTS[@]}"; do
+  if [[ "$kv" != *=* ]]; then
+    echo "Invalid --input value: ${kv} (expected key=value)" >&2
+    exit 2
+  fi
+  key="${kv%%=*}"
+  value="${kv#*=}"
+  append_input_field "$key" "$value"
+done
+
+PAYLOAD=$(cat <<EOF
+{
+  "ref": "${TARGET_REF}",
+  "inputs": {
+${PAYLOAD_INPUTS}
+  }
+}
+EOF
+)
+
+ensure_curl
+
 echo "Dispatching ${WORKFLOW_FILE} for ${HEAD_SHA} on ${TARGET_REF}"
-curl -fsS -X POST \
-  -H "Authorization: token ${TOKEN}" \
-  -H "Content-Type: application/json" \
-  "${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/${WORKFLOW_FILE}/dispatches" \
-  -d "${PAYLOAD}"
+echo "trace_id=${TRACE_ID}"
+
+dispatched=false
+attempt=0
+total_attempts=$((MAX_ROUNDS * ${#API_BASES[@]}))
+
+for round in $(seq 1 "${MAX_ROUNDS}"); do
+  for api_base in "${API_BASES[@]}"; do
+    attempt=$((attempt + 1))
+    dispatch_url="${api_base}/repos/${REPO_OWNER}/${REPO_NAME}/actions/workflows/${WORKFLOW_FILE}/dispatches"
+    response_file="/tmp/dispatch-response.txt"
+    : > "${response_file}"
+
+    echo "dispatch_attempt=${attempt}/${total_attempts}"
+    echo "dispatch_round=${round}/${MAX_ROUNDS}"
+    echo "dispatch_api_base=${api_base}"
+
+    dispatch_start_ms=$(date +%s%3N)
+    http_code=$(curl -sS --connect-timeout 5 --max-time 20 -o "${response_file}" -w "%{http_code}" -X POST \
+      -H "Authorization: token ${TOKEN}" \
+      -H "Content-Type: application/json" \
+      "${dispatch_url}" \
+      -d "${PAYLOAD}" || true)
+    dispatch_end_ms=$(date +%s%3N)
+    dispatch_elapsed_ms=$((dispatch_end_ms - dispatch_start_ms))
+
+    echo "dispatch_elapsed_ms=${dispatch_elapsed_ms}"
+    echo "dispatch_http_code=${http_code}"
+
+    if [[ "${http_code}" =~ ^[0-9]+$ ]] && [[ "${http_code}" -ge 200 ]] && [[ "${http_code}" -lt 300 ]]; then
+      echo "Dispatch succeeded"
+      dispatched=true
+      break 2
+    fi
+
+    echo "Dispatch attempt failed via ${api_base} (HTTP ${http_code})"
+    if [[ -s "${response_file}" ]]; then
+      echo "Response body:"
+      cat "${response_file}" || true
+    fi
+  done
+
+  sleep_seconds=$((2 ** round))
+  echo "Dispatch round ${round} failed; backing off for ${sleep_seconds}s"
+  sleep "${sleep_seconds}"
+done
+
+if [[ "${dispatched}" != "true" ]]; then
+  echo "Dispatch failed on all endpoints after ${MAX_ROUNDS} rounds" >&2
+  exit 1
+fi