From 524bf21244fd024b0cf1736c2ff8f3de75be289c Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Sun, 26 Oct 2025 21:34:06 -0400 Subject: [PATCH] Redoing this now with multi stage docker images for the CICD. Signed-off-by: Cliff Hill --- .gitea/workflows/tests.yml | 411 +++++++++---------------------------- Dockerfile.cicd-backend | 44 ++++ Dockerfile.cicd-frontend | 33 +++ Dockerfile.cicd-setup | 24 +++ 4 files changed, 194 insertions(+), 318 deletions(-) create mode 100644 Dockerfile.cicd-backend create mode 100644 Dockerfile.cicd-frontend create mode 100644 Dockerfile.cicd-setup diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 21f4538..18aaf6c 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -8,79 +8,21 @@ on: jobs: setup: - name: Setup and Checkout + name: Build Base Setup Image runs-on: ubuntu-act steps: - - name: Runner Info - run: | - echo "=== Setup Job - $(date) ===" - - # Debug available runner variables - echo "=== Runner Detection Debug ===" - echo "HOSTNAME: ${HOSTNAME:-not_set}" - echo "RUNNER_NAME: ${RUNNER_NAME:-not_set}" - echo "GITEA_RUNNER_NAME: ${GITEA_RUNNER_NAME:-not_set}" - echo "ACT_RUNNER_NAME: ${ACT_RUNNER_NAME:-not_set}" - echo "RUNNER_ENVIRONMENT: ${RUNNER_ENVIRONMENT:-not_set}" - echo "ACTIONS_RUNNER_NAME: ${ACTIONS_RUNNER_NAME:-not_set}" - - echo "=== Additional Runner Variables ===" - echo "RUNNER_LABEL: ${RUNNER_LABEL:-not_set}" - echo "RUNNER_VERSION: ${RUNNER_VERSION:-not_set}" - echo "GITEA_ACTIONS_RUNNER_VERSION: ${GITEA_ACTIONS_RUNNER_VERSION:-not_set}" - echo "CONTAINER_NAME: ${CONTAINER_NAME:-not_set}" - - echo "=== Docker/Container Info ===" - echo "Container hostname: $(hostname)" - cat /etc/hostname 2>/dev/null || echo "No /etc/hostname" - - echo "=== All Environment Variables ===" - env | grep -iE "(runner|gitea|act|container)" | head -20 - - # Try to detect actual runner from compose service or docker labels - if command -v docker >/dev/null 2>&1; then - echo "=== Docker Container Labels ===" - docker inspect $(hostname) --format='{{range $key, $value := .Config.Labels}}{{$key}}={{$value}}{{"\n"}}{{end}}' 2>/dev/null | grep -i runner || echo "No runner labels found" - fi - - # Detect runner name with better logic - DETECTED_RUNNER="${GITEA_RUNNER_NAME:-${RUNNER_NAME:-${ACT_RUNNER_NAME:-${HOSTNAME:-unknown}}}}" - - # Try to extract runner from job container name - if [ -n "${JOB_CONTAINER_NAME}" ]; then - echo "Job Container: ${JOB_CONTAINER_NAME}" - # Extract task ID from container name (e.g., GITEA-ACTIONS-TASK-249) - TASK_ID=$(echo "${JOB_CONTAINER_NAME}" | grep -o 'TASK-[0-9]*' || echo "") - if [ -n "${TASK_ID}" ]; then - echo "Task ID: ${TASK_ID}" - fi - fi - - # Check if we're on a specific Pi by looking at network or other clues - echo "Network interfaces:" - ip addr show | grep -E "inet.*eth|inet.*wlan" | head -3 || echo "No network info available" - - echo "Detected Runner: $DETECTED_RUNNER" # Show system info - echo "Container ID: $(hostname)" - echo "Architecture: $(uname -m)" - - name: Checkout code - # Skip actions/checkout@v4 due to SSL issues with dogar.darkhelm.org - # Use manual checkout with multiple fallback methods env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} GITHUB_WORKSPACE: ${{ github.workspace }} GITHUB_SHA: ${{ github.sha }} run: | echo "=== Repository Checkout ===" cd "${GITHUB_WORKSPACE}" - - # Clean workspace rm -rf ./* .git 2>/dev/null || true - # Set up SSH key from Gitea repository secret + # Set up SSH key if [ -n "${SSH_PRIVATE_KEY}" ]; then mkdir -p ~/.ssh echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_rsa @@ -88,315 +30,148 @@ jobs: ssh-keyscan -p 2222 dogar.darkhelm.org >> ~/.ssh/known_hosts 2>/dev/null fi - # Clone repository via SSH + # Clone repository GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no" \ git clone --depth 1 --branch main \ ssh://git@dogar.darkhelm.org:2222/DarkHelm.org/plex-playlist.git . - # Checkout specific commit if provided if [ -n "${GITHUB_SHA}" ]; then git checkout "${GITHUB_SHA}" 2>/dev/null || echo "Using main branch HEAD" fi - echo "✓ Repository checkout completed" - - name: Debug artifact connectivity + - name: Build cicd-setup image run: | - echo "=== Artifact Service Debug ===" - echo "ACTIONS_RUNTIME_URL: ${ACTIONS_RUNTIME_URL}" - echo "ACTIONS_CACHE_URL: ${ACTIONS_CACHE_URL}" + echo "=== Building CICD Setup Image ===" - # Test connectivity to artifact services - echo "Testing artifact service connectivity..." - if [ -n "${ACTIONS_CACHE_URL}" ]; then - echo "Testing cache URL: ${ACTIONS_CACHE_URL}" - curl -v --connect-timeout 5 --max-time 10 "${ACTIONS_CACHE_URL}" 2>&1 || echo "Cache URL test failed" - fi + # Build the base setup image with source code + docker build -f Dockerfile.cicd-setup -t cicd-setup:latest . - if [ -n "${ACTIONS_RUNTIME_URL}" ]; then - echo "Testing runtime URL: ${ACTIONS_RUNTIME_URL}" - curl -v --connect-timeout 5 --max-time 10 "${ACTIONS_RUNTIME_URL}" 2>&1 || echo "Runtime URL test failed" - fi + # Verify the image was built + docker images | grep cicd-setup - - name: Prepare source for upload - run: | - echo "=== Preparing Source Code for Upload ===" - # Show what files we're uploading - echo "Files to be uploaded:" - find . -name "*.yml" -o -name "*.yaml" -o -name "*.json" -o -name "*.md" -o -name "*.py" -o -name "*.toml" -o -name "Dockerfile*" -o -name "compose*.yml" -o -path "./backend/*" -o -path "./frontend/*" -o -path "./scripts/*" | grep -v ".git" | sort - - # Show total size - echo "Total size: $(du -sh . | cut -f1)" - echo "File count: $(find . -type f | wc -l) files" - - - name: Upload source code - uses: actions/upload-artifact@v3 - with: - name: source-code - path: | - backend/ - frontend/ - scripts/ - *.yml - *.yaml - *.json - *.md - *.py - *.toml - Dockerfile* - compose*.yml - !.git - !**/node_modules/ - !**/__pycache__/ - !**/.pytest_cache/ - !**/dist/ - !**/build/ - retention-days: 1 + echo "✓ CICD setup image ready" backend-setup: - name: Backend Setup (Python 3.13 + uv + Environment) + name: Build Backend Environment runs-on: ubuntu-act needs: setup steps: - - name: Runner Info + - name: Build cicd-backend image run: | - echo "=== Backend Setup - $(date) ===" - RUNNER="${GITEA_RUNNER_NAME:-${RUNNER_NAME:-${ACT_RUNNER_NAME:-$(hostname)}}}" - echo "Runner: $RUNNER | User: $(whoami) | OS: $(uname -s)" + echo "=== Building CICD Backend Image ===" - - name: Download source code - uses: actions/download-artifact@v3 - with: - name: source-code - path: . + # Build the backend image extending the setup image + docker build -f Dockerfile.cicd-backend -t cicd-backend:latest . - - name: Install Python 3.13 - run: | - echo "=== Installing Python 3.13 ===" - sudo apt-get update -qq - sudo apt-get install -y software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt-get update -qq - sudo apt-get install -y python3.13 python3.13-venv python3.13-dev curl git + # Verify the image was built and tools are available + docker images | grep cicd-backend + docker run --rm cicd-backend:latest bash -c "cd /workspace/backend && ruff --version && pyright --version" - - name: Install uv - run: | - echo "=== Installing uv Package Manager ===" - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Verify installations - run: | - echo "=== Verifying Python and uv Installations ===" - export PATH="$HOME/.cargo/bin:$PATH" - python3.13 --version - uv --version - echo "✓ Python 3.13 and uv successfully installed" - - - name: Setup Python environment with uv - working-directory: ./backend - run: | - export PATH="$HOME/.cargo/bin:$PATH" - echo "=== Setting up Python environment ===" - - # Validate pyproject.toml exists - [ -f pyproject.toml ] || { echo "❌ pyproject.toml not found"; exit 1; } - - # Create virtual environment and install dependencies - uv venv .venv --python python3.13 - . .venv/bin/activate - uv pip install -e ".[dev]" - - # Verify installation - python --version - python -c "import backend; print(f'Backend version: {backend.__version__}')" - echo "✓ Backend environment setup complete" - - - name: Upload backend environment - uses: actions/upload-artifact@v3 - with: - name: backend-environment - path: | - . - !.git - retention-days: 1 + echo "✓ CICD backend image ready" frontend-setup: - name: Frontend Setup (Node.js 24 + Yarn Berry + Build) + name: Build Frontend Environment runs-on: ubuntu-act needs: setup steps: - - name: Runner Info + - name: Build cicd-frontend image run: | - echo "=== Frontend Setup - $(date) ===" - RUNNER="${GITEA_RUNNER_NAME:-${RUNNER_NAME:-${ACT_RUNNER_NAME:-$(hostname)}}}" - echo "Runner: $RUNNER | User: $(whoami) | OS: $(uname -s)" + echo "=== Building CICD Frontend Image ===" - - name: Download source code - uses: actions/download-artifact@v3 - with: - name: source-code - path: . + # Build the frontend image extending the setup image + docker build -f Dockerfile.cicd-frontend -t cicd-frontend:latest . - - name: Install Node.js 24 - run: | - echo "=== Installing Node.js 24 ===" - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - - sudo apt-get install -y nodejs + # Verify the image was built and tools are available + docker images | grep cicd-frontend + docker run --rm cicd-frontend:latest bash -c "cd /workspace/frontend && yarn eslint --version && ls -la dist/" - - name: Setup Yarn Berry - run: | - echo "=== Setting up Yarn Berry ===" - corepack enable - corepack prepare yarn@stable --activate - - - name: Verify Node and Yarn - run: | - echo "=== Verifying Node.js and Yarn Installations ===" - node --version - yarn --version - echo "✓ Node.js 24 and Yarn Berry successfully installed" - - - name: Install dependencies - working-directory: ./frontend - timeout-minutes: 10 - run: | - echo "=== Frontend Dependency Installation ===" - - # Setup Yarn Berry with lockfile creation allowed - yarn set version stable - cat > .yarnrc.yml << EOF - enableImmutableInstalls: false - nodeLinker: pnp - compressionLevel: 0 - httpTimeout: 60000 - EOF - - # Add packageManager field if missing - if ! grep -q '"packageManager"' package.json; then - sed -i '$ s/}/ "packageManager": "yarn@4.10.3",\n}/' package.json - fi - - # Install dependencies - yarn install - echo "✓ Frontend dependencies installed" - - - name: Verify and build frontend - working-directory: ./frontend - run: | - echo "=== TypeScript Check & Build ===" - - # Check TypeScript compilation - yarn run tsc --noEmit || echo "TypeScript check completed with warnings" - - # Build the frontend - yarn run build - - # Verify build output - [ -d "dist" ] || { echo "❌ Build failed - no dist directory"; exit 1; } - echo "✓ Build successful ($(du -sh dist/ | cut -f1))" - - - name: Upload frontend environment - uses: actions/upload-artifact@v3 - with: - name: frontend-environment - path: | - . - !.git - retention-days: 1 + echo "✓ CICD frontend image ready" backend-tests: - name: Backend Tests (Python 3.13 + uv) + name: Backend Tests runs-on: ubuntu-act needs: backend-setup steps: - - name: Runner Info + - name: Run backend tests run: | - echo "=== Backend Tests - $(date) ===" - RUNNER="${GITEA_RUNNER_NAME:-${RUNNER_NAME:-${ACT_RUNNER_NAME:-$(hostname)}}}" - echo "Runner: $RUNNER | User: $(whoami)" - - - name: Download backend environment - uses: actions/download-artifact@v3 - with: - name: backend-environment - path: . - - - name: Restore Python environment - run: | - echo "=== Restoring Python Environment ===" - export PATH="$HOME/.cargo/bin:$PATH" - cd backend - - [ -d ".venv" ] || { echo "❌ Virtual environment not found"; exit 1; } - . .venv/bin/activate - echo "✓ Python $(python --version | cut -d' ' -f2) environment restored" - - - name: Run tests with pytest - working-directory: ./backend - run: | - export PATH="$HOME/.cargo/bin:$PATH" - . .venv/bin/activate - echo "=== Running Backend Tests ===" - # Run pytest with automatic typeguard hooks and coverage - python -m pytest tests/ -v \ - --cov=backend \ - --cov-report=term-missing \ - --cov-report=xml \ - --cov-fail-under=95 + # Run tests using the pre-built backend image + docker run --rm cicd-backend:latest bash -c " + cd /workspace/backend && + uv run pytest -v --tb=short --cov=src --cov-report=term-missing + " - echo "✓ Backend tests completed with automatic typeguard hooks and 95% coverage!" + echo "✓ Backend tests completed" + + backend-lint: + name: Backend Linting + runs-on: ubuntu-act + needs: backend-setup + + steps: + - name: Run backend linting + run: | + echo "=== Running Backend Linting ===" + + # Run all backend linting tools using the pre-built image + docker run --rm cicd-backend:latest bash -c " + cd /workspace/backend && + echo '--- Running ruff linting ---' && + uv run ruff check . && + echo '--- Running ruff formatting check ---' && + uv run ruff format --check . && + echo '--- Running pyright type checking ---' && + uv run pyright && + echo '--- Running darglint docstring linting ---' && + uv run darglint src/ + " + + echo "✓ Backend linting completed" frontend-tests: - name: Frontend Tests (TypeScript + Vue + Yarn Berry) + name: Frontend Tests runs-on: ubuntu-act needs: frontend-setup steps: - - name: Runner Info - run: | - echo "=== Frontend Tests - $(date) ===" - RUNNER="${GITEA_RUNNER_NAME:-${RUNNER_NAME:-${ACT_RUNNER_NAME:-$(hostname)}}}" - echo "Runner: $RUNNER | User: $(whoami)" - - - name: Download frontend environment - uses: actions/download-artifact@v3 - with: - name: frontend-environment - path: . - - - name: Restore Node environment - run: | - echo "=== Restoring Node.js Environment ===" - - # Install Node.js and corepack if not available - if ! command -v node &> /dev/null; then - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - - sudo apt-get install -y nodejs - fi - - if ! command -v corepack &> /dev/null; then - sudo npm install -g corepack - corepack enable - fi - - # Verify environment and build artifacts - cd frontend - echo "✓ Node $(node --version) | Yarn $(yarn --version)" - [ -d "dist" ] || { echo "❌ Frontend build not found"; exit 1; } - echo "✓ Build artifacts found ($(du -sh dist/ | cut -f1))" - - name: Run frontend tests - working-directory: ./frontend run: | echo "=== Running Frontend Tests ===" - # Run Vitest with automatic Zod validation hooks and coverage - yarn run test --coverage --coverage.enabled=true --coverage.thresholds.lines=85 --coverage.thresholds.functions=85 --coverage.thresholds.branches=85 --coverage.thresholds.statements=85 + # Run tests using the pre-built frontend image + docker run --rm cicd-frontend:latest bash -c " + cd /workspace/frontend && + yarn test --run --reporter=verbose + " - echo "✓ Frontend tests completed with automatic Zod validation hooks and 85% coverage!" + echo "✓ Frontend tests completed" + + frontend-lint: + name: Frontend Linting + runs-on: ubuntu-act + needs: frontend-setup + + steps: + - name: Run frontend linting + run: | + echo "=== Running Frontend Linting ===" + + # Run all frontend linting tools using the pre-built image + docker run --rm cicd-frontend:latest bash -c " + cd /workspace/frontend && + echo '--- Running ESLint ---' && + yarn lint && + echo '--- Running Prettier check ---' && + yarn format:check && + echo '--- Running TypeScript type checking ---' && + yarn type-check && + echo '--- Running TSDoc linting ---' && + yarn lint:tsdoc + " + + echo "✓ Frontend linting completed" diff --git a/Dockerfile.cicd-backend b/Dockerfile.cicd-backend new file mode 100644 index 0000000..c0565b4 --- /dev/null +++ b/Dockerfile.cicd-backend @@ -0,0 +1,44 @@ +# CICD Backend - Extends setup with Python environment +FROM cicd-setup:latest + +# Install Python 3.13 and system dependencies +RUN apt-get update && apt-get install -y \ + software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt-get update && apt-get install -y \ + python3.13 \ + python3.13-venv \ + python3.13-dev \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install uv package manager +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv + +# Set working directory to backend +WORKDIR /workspace/backend + +# Create and activate virtual environment +ENV VIRTUAL_ENV=/workspace/backend/.venv +RUN uv venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Install backend dependencies in dev mode (includes ruff, pyright, darglint, etc.) +RUN uv sync --dev + +# Install the backend package in development mode +RUN uv pip install -e . + +# Set Python path for imports +ENV PYTHONPATH=/workspace/backend/src:/workspace/backend + +# Verify tools are available +RUN which ruff && ruff --version +RUN which pyright && pyright --version +RUN which darglint && darglint --version + +# Set working directory back to workspace root for flexibility +WORKDIR /workspace + +# Default command +CMD ["/bin/bash"] diff --git a/Dockerfile.cicd-frontend b/Dockerfile.cicd-frontend new file mode 100644 index 0000000..3f26732 --- /dev/null +++ b/Dockerfile.cicd-frontend @@ -0,0 +1,33 @@ +# CICD Frontend - Extends setup with Node.js environment +FROM cicd-setup:latest + +# Install Node.js 20 +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Enable corepack for yarn +RUN corepack enable + +# Set working directory to frontend +WORKDIR /workspace/frontend + +# Install frontend dependencies +RUN yarn install --immutable + +# Verify tools are available (these should be installed via package.json) +RUN yarn eslint --version +RUN yarn prettier --version +RUN yarn tsc --version + +# Build the application +RUN yarn build + +# Verify build output +RUN ls -la dist/ && echo "Frontend build completed successfully" + +# Set working directory back to workspace root for flexibility +WORKDIR /workspace + +# Default command +CMD ["/bin/bash"] diff --git a/Dockerfile.cicd-setup b/Dockerfile.cicd-setup new file mode 100644 index 0000000..7cb4406 --- /dev/null +++ b/Dockerfile.cicd-setup @@ -0,0 +1,24 @@ +# CICD Setup - Base image with source code checkout +FROM ubuntu:22.04 + +# Install essential tools +RUN apt-get update && apt-get install -y \ + git \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /workspace + +# Copy the entire project (this will be the "checkout" layer) +COPY . . + +# Ensure proper permissions +RUN chmod -R 755 /workspace + +# Default to bash for better shell support +SHELL ["/bin/bash", "-c"] + +# This image contains the full project source code +CMD ["/bin/bash"]