Making network connectivity more resiliant and getting the secrets more secure.

Signed-off-by: Cliff Hill <xlorep@darkhelm.org>

.gitea/workflows/cicd.yml

@@ -18,7 +18,7 @@ jobs:
         run: |
           echo "=== Minimal Repository Checkout for Dockerfile ==="
 
-          # Set up SSH key
+          # Set up SSH key securely (temporary file approach)
           if [ -n "${SSH_PRIVATE_KEY}" ]; then
             mkdir -p ~/.ssh
             echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa
@@ -31,10 +31,13 @@ jobs:
             git clone --depth 1 --no-checkout \
             ssh://git@dogar.darkhelm.org:2222/DarkHelm.org/plex-playlist.git .
 
-          # Checkout only the Dockerfile
-          git checkout HEAD -- Dockerfile.cicd
+          # Checkout only the Dockerfile and dockerignore
+          git checkout HEAD -- Dockerfile.cicd .dockerignore
 
-          echo "✓ Dockerfile.cicd ready for build"
+          # Clean up SSH key for security
+          rm -f ~/.ssh/id_rsa
+
+          echo "✓ Dockerfile.cicd ready for secure build"
 
       - name: Build and push CICD image
         env:
@@ -42,15 +45,25 @@ jobs:
           SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
           GITHUB_SHA: ${{ github.sha }}
         run: |
-          echo "=== Building CICD Image ==="
+          echo "=== Building CICD Image with Secure Secrets ==="
 
-          # Build CICD image with all tools, code, and dependencies
-          # Pass SSH key as build arg so Dockerfile can checkout repo
+          # Create temporary SSH key file for BuildKit secrets
+          echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
+          chmod 600 /tmp/ssh_key
+
+          # Enable Docker BuildKit for secrets support
+          export DOCKER_BUILDKIT=1
+
+          # Build CICD image using secure BuildKit secrets
+          # SSH key is mounted securely and never stored in image layers
           docker build -f Dockerfile.cicd \
-            --build-arg SSH_PRIVATE_KEY="$SSH_PRIVATE_KEY" \
+            --secret id=ssh_private_key,src=/tmp/ssh_key \
             --build-arg GITHUB_SHA="$GITHUB_SHA" \
             -t cicd:latest .
 
+          # Clean up temporary SSH key file
+          rm -f /tmp/ssh_key
+
           # Tag for Gitea container registry
           docker tag cicd:latest dogar.darkhelm.org/darkhelm.org/plex-playlist/cicd:latest
           docker tag cicd:latest dogar.darkhelm.org/darkhelm.org/plex-playlist/cicd:${GITHUB_SHA:-latest}

Dockerfile.cicd

@@ -16,11 +16,17 @@ RUN apt-get update && apt-get install -y \
     apt-fast \
     && rm -rf /var/lib/apt/lists/*
 
-# Configure apt-fast for non-interactive use
+# Configure apt-fast for non-interactive use with timeouts
 RUN echo 'apt-fast apt-fast/maxdownloads string 10' | debconf-set-selections && \
     echo 'apt-fast apt-fast/dlflag boolean true' | debconf-set-selections && \
     echo 'apt-fast apt-fast/aptmanager string apt-get' | debconf-set-selections
 
+# Configure apt timeouts and retries
+RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \
+    echo 'Acquire::http::Timeout "60";' >> /etc/apt/apt.conf.d/80retries && \
+    echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80retries && \
+    echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80retries
+
 # Install system dependencies using apt-fast
 RUN apt-fast update && apt-fast install -y \
     git \
@@ -32,18 +38,37 @@ RUN apt-fast update && apt-fast install -y \
     tzdata \
     && rm -rf /var/lib/apt/lists/*
 
-# Install Python 3.13
-RUN add-apt-repository -y ppa:deadsnakes/ppa \
-    && apt-fast update && apt-fast install -y \
-    python3.13 \
-    python3.13-venv \
-    python3.13-dev \
-    && rm -rf /var/lib/apt/lists/*
+# Install Python 3.13 with retry and fallback mechanisms
+RUN for i in 1 2 3; do \
+        echo "Attempt $i: Adding deadsnakes PPA..." && \
+        add-apt-repository -y ppa:deadsnakes/ppa && \
+        apt-get update --timeout=60 && \
+        break || \
+        (echo "Attempt $i failed, retrying in 10s..." && sleep 10); \
+    done
 
-# Install Node.js 24
-RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
-    && apt-fast update && apt-fast install -y nodejs \
-    && rm -rf /var/lib/apt/lists/*
+RUN for i in 1 2 3; do \
+        echo "Attempt $i: Installing Python 3.13..." && \
+        apt-fast install -y --timeout=300 \
+        python3.13 \
+        python3.13-venv \
+        python3.13-dev && \
+        break || \
+        (echo "Attempt $i failed, retrying in 15s..." && sleep 15); \
+    done && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Node.js 24 with retry mechanism
+RUN for i in 1 2 3; do \
+        echo "Attempt $i: Installing Node.js 24..." && \
+        curl -fsSL --connect-timeout 30 --max-time 300 \
+        https://deb.nodesource.com/setup_24.x | bash - && \
+        apt-fast update --timeout=60 && \
+        apt-fast install -y --timeout=300 nodejs && \
+        break || \
+        (echo "Attempt $i failed, retrying in 15s..." && sleep 15); \
+    done && \
+    rm -rf /var/lib/apt/lists/*
 
 # Enable corepack for yarn and set up Yarn Berry
 RUN corepack enable \
@@ -53,35 +78,30 @@ RUN corepack enable \
 # Install uv package manager globally
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 
-# Accept build arguments for Git checkout
-ARG SSH_PRIVATE_KEY
+# Accept build arguments for Git checkout (no secrets here!)
 ARG GITHUB_SHA
 
 # Set working directory
 WORKDIR /workspace
 
-# Set up SSH and clone repository
-RUN if [ -n "$SSH_PRIVATE_KEY" ]; then \
-        mkdir -p ~/.ssh && \
-        echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa && \
-        chmod 600 ~/.ssh/id_rsa && \
-        echo "Host *" > ~/.ssh/config && \
-        echo "    StrictHostKeyChecking no" >> ~/.ssh/config && \
-        echo "    UserKnownHostsFile /dev/null" >> ~/.ssh/config && \
-        chmod 600 ~/.ssh/config && \
-        ssh-keyscan -p 2222 dogar.darkhelm.org >> ~/.ssh/known_hosts 2>/dev/null; \
-    fi
-
-# Clone repository
-RUN GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+# Set up SSH and clone repository using BuildKit secrets
+RUN --mount=type=secret,id=ssh_private_key \
+    mkdir -p ~/.ssh && \
+    cp /run/secrets/ssh_private_key ~/.ssh/id_rsa && \
+    chmod 600 ~/.ssh/id_rsa && \
+    echo "Host dogar.darkhelm.org" > ~/.ssh/config && \
+    echo "    Port 2222" >> ~/.ssh/config && \
+    echo "    StrictHostKeyChecking no" >> ~/.ssh/config && \
+    echo "    UserKnownHostsFile /dev/null" >> ~/.ssh/config && \
+    chmod 600 ~/.ssh/config && \
+    ssh-keyscan -p 2222 dogar.darkhelm.org >> ~/.ssh/known_hosts 2>/dev/null && \
+    GIT_SSH_COMMAND="ssh -F ~/.ssh/config" \
     git clone --depth 1 --branch main \
     ssh://git@dogar.darkhelm.org:2222/DarkHelm.org/plex-playlist.git . && \
     if [ -n "$GITHUB_SHA" ]; then \
         git checkout "$GITHUB_SHA" 2>/dev/null || echo "Using main branch HEAD"; \
-    fi
-
-# Clean up SSH key for security
-RUN rm -rf ~/.ssh
+    fi && \
+    rm -rf ~/.ssh
 
 # Set up Python environment for backend
 WORKDIR /workspace/backend
@@ -117,13 +137,17 @@ RUN cd /workspace/frontend && \
     yarn tsc --version && \
     yarn vitest --version
 
-# Create a script to set up SSH for git operations (if needed for updates)
+# Create a script to set up SSH for git operations (using secrets mount)
 RUN echo '#!/bin/bash' > /usr/local/bin/setup-ssh && \
-    echo 'mkdir -p ~/.ssh' >> /usr/local/bin/setup-ssh && \
-    echo 'echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa' >> /usr/local/bin/setup-ssh && \
-    echo 'chmod 600 ~/.ssh/id_rsa' >> /usr/local/bin/setup-ssh && \
-    echo 'ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null' >> /usr/local/bin/setup-ssh && \
-    echo 'ssh-keyscan -H dogar.darkhelm.org >> ~/.ssh/known_hosts 2>/dev/null' >> /usr/local/bin/setup-ssh && \
+    echo 'if [ -f /run/secrets/ssh_private_key ]; then' >> /usr/local/bin/setup-ssh && \
+    echo '  mkdir -p ~/.ssh' >> /usr/local/bin/setup-ssh && \
+    echo '  cp /run/secrets/ssh_private_key ~/.ssh/id_rsa' >> /usr/local/bin/setup-ssh && \
+    echo '  chmod 600 ~/.ssh/id_rsa' >> /usr/local/bin/setup-ssh && \
+    echo '  ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null' >> /usr/local/bin/setup-ssh && \
+    echo '  ssh-keyscan -p 2222 -H dogar.darkhelm.org >> ~/.ssh/known_hosts 2>/dev/null' >> /usr/local/bin/setup-ssh && \
+    echo 'else' >> /usr/local/bin/setup-ssh && \
+    echo '  echo "No SSH key provided via secrets mount"' >> /usr/local/bin/setup-ssh && \
+    echo 'fi' >> /usr/local/bin/setup-ssh && \
     chmod +x /usr/local/bin/setup-ssh
 
 # Set Python path for backend

docs/SECURE_DOCKER_CICD.md

@@ -0,0 +1,107 @@
+# Secure Docker CI/CD with BuildKit Secrets
+
+This document explains how our CI/CD pipeline securely handles SSH keys using Docker BuildKit secrets.
+
+## 🔒 Security Benefits
+
+### Before (Insecure)
+```dockerfile
+ARG SSH_PRIVATE_KEY
+RUN echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
+```
+- ❌ SSH key stored in Docker image layers
+- ❌ Visible in `docker history`
+- ❌ Can be extracted from images
+- ❌ Security vulnerability
+
+### After (Secure)
+```dockerfile
+RUN --mount=type=secret,id=ssh_private_key \
+    cp /run/secrets/ssh_private_key ~/.ssh/id_rsa && \
+    # ... use key ... && \
+    rm -rf ~/.ssh
+```
+- ✅ SSH key never stored in image layers
+- ✅ Not visible in `docker history`
+- ✅ Cannot be extracted from final image
+- ✅ Secure by design
+
+## 🏗️ CI/CD Pipeline Implementation
+
+### Gitea Actions Workflow
+The `.gitea/workflows/cicd.yml` file now uses:
+
+1. **Docker BuildKit Enabled**
+   ```yaml
+   export DOCKER_BUILDKIT=1
+   ```
+
+2. **Secure Secret Mounting**
+   ```yaml
+   # Create temporary SSH key file
+   echo "${SSH_PRIVATE_KEY}" > /tmp/ssh_key
+   chmod 600 /tmp/ssh_key
+
+   # Build with secret mount
+   docker build -f Dockerfile.cicd \
+     --secret id=ssh_private_key,src=/tmp/ssh_key \
+     -t cicd:latest .
+
+   # Clean up immediately
+   rm -f /tmp/ssh_key
+   ```
+
+### Local Development
+Use the secure build script:
+```bash
+./scripts/build-cicd-secure.sh plex-playlist-cicd:latest
+```
+
+## 🔧 Required Setup
+
+### 1. Gitea Secrets Configuration
+Ensure these secrets are configured in your Gitea repository:
+- `SSH_PRIVATE_KEY`: Your private SSH key for git operations
+- `GITEA_TOKEN`: Token for pushing to container registry
+
+### 2. Docker BuildKit Support
+- **Gitea Actions**: Automatically enabled with `DOCKER_BUILDKIT=1`
+- **Local builds**: Requires Docker 18.09+ with BuildKit enabled
+- **CI runners**: Ensure BuildKit support in your runner environment
+
+## 📋 Best Practices Applied
+
+1. **Temporary Files**: SSH keys are written to temporary files and immediately removed
+2. **File Permissions**: SSH keys get proper 600 permissions
+3. **Multi-stage Security**: Keys are cleaned up within the same RUN command
+4. **No ARG Secrets**: Never pass secrets via build arguments
+5. **Dockerignore Protection**: `.dockerignore` prevents accidental secret inclusion
+
+## 🧪 Testing Security
+
+Verify no secrets in image:
+```bash
+# Build the image
+./scripts/build-cicd-secure.sh test-image
+
+# Check history (should show no secrets)
+docker history test-image
+
+# Try to find SSH keys (should find none)
+docker run --rm test-image find / -name "*rsa*" -o -name "*ssh*" 2>/dev/null
+```
+
+## 🚀 Migration Checklist
+
+- [x] Updated Dockerfile.cicd to use `--mount=type=secret`
+- [x] Removed insecure `ARG SSH_PRIVATE_KEY`
+- [x] Modified CI/CD workflow to use BuildKit secrets
+- [x] Added `.dockerignore` patterns for SSH keys
+- [x] Created secure build script
+- [x] Documented security improvements
+
+## 📚 References
+
+- [Docker BuildKit Secrets](https://docs.docker.com/build/building/secrets/)
+- [Docker Security Best Practices](https://docs.docker.com/develop/security-best-practices/)
+- [Gitea Actions Documentation](https://docs.gitea.com/usage/actions/)

scripts/build-cicd-secure.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Secure Docker build script using BuildKit secrets
+# Usage: ./scripts/build-cicd-secure.sh [image-tag]
+
+set -e
+
+# Default image tag
+IMAGE_TAG="${1:-plex-playlist-cicd:latest}"
+
+# Check if SSH key exists
+SSH_KEY_PATH="${SSH_KEY_PATH:-$HOME/.ssh/id_rsa}"
+if [ ! -f "$SSH_KEY_PATH" ]; then
+    echo "Error: SSH private key not found at $SSH_KEY_PATH"
+    echo "Set SSH_KEY_PATH environment variable or ensure key exists at default location"
+    exit 1
+fi
+
+echo "Building Docker image with secure secrets handling..."
+echo "Image tag: $IMAGE_TAG"
+echo "SSH key: $SSH_KEY_PATH"
+
+# Enable Docker BuildKit (required for --secret)
+export DOCKER_BUILDKIT=1
+
+# Build with secrets mount - SSH key never enters image layers
+docker build \
+    --secret id=ssh_private_key,src="$SSH_KEY_PATH" \
+    --build-arg GITHUB_SHA="${GITHUB_SHA:-}" \
+    --file Dockerfile.cicd \
+    --tag "$IMAGE_TAG" \
+    .
+
+echo "✅ Build completed successfully!"
+echo "Run with: docker run -it $IMAGE_TAG"
+
+# Optional: Test the image
+if [ "${TEST_IMAGE:-}" = "true" ]; then
+    echo "🧪 Testing image..."
+    docker run --rm "$IMAGE_TAG" bash -c "
+        echo 'Testing tools...' &&
+        python3 --version &&
+        node --version &&
+        yarn --version &&
+        git --version &&
+        echo '✅ All tools working!'
+    "
+fi