diff --git a/Dockerfile.cicd-base b/Dockerfile.cicd-base index 3663129..d240590 100644 --- a/Dockerfile.cicd-base +++ b/Dockerfile.cicd-base @@ -138,7 +138,8 @@ RUN echo "=== Installing Playwright Browsers ===" && \ RUN echo "=== Pre-installing Common Python Dev Tools ===" && \ # Create a global uv environment with common dev tools uv venv /opt/python-dev-tools && \ - /opt/python-dev-tools/bin/uv pip install \ + # Use system uv to install packages into the virtual environment + uv pip install --python /opt/python-dev-tools/bin/python \ ruff>=0.6.0 \ pyright>=1.1.380 \ pytest>=7.4.0 \ @@ -150,7 +151,8 @@ RUN echo "=== Pre-installing Common Python Dev Tools ===" && \ yamllint>=1.35.0 \ toml-sort>=0.23.0 \ xdoctest>=1.1.0 \ - language-formatters-pre-commit-hooks>=2.14.0 && \ + language-formatters-pre-commit-hooks>=2.14.0 \ + poethepoet>=0.24.0 && \ echo "✓ Common Python dev tools pre-installed in /opt/python-dev-tools" # Create a script to set up SSH for git operations (using secrets mount) @@ -182,6 +184,7 @@ RUN echo "=== Base System Verification ===" && \ /opt/python-dev-tools/bin/pyright --version && \ /opt/python-dev-tools/bin/pytest --version && \ /opt/python-dev-tools/bin/pre-commit --version && \ + /opt/python-dev-tools/bin/poe --version && \ echo "✓ All base system and development tools verified successfully" # Set working directory diff --git a/README.md b/README.md index e1ee4b2..197425c 100644 --- a/README.md +++ b/README.md @@ -32,33 +32,79 @@ This project uses comprehensive linting and formatting: - `vue-tsc` - Vue TypeScript checking - `eslint-plugin-tsdoc` - TSDoc documentation linting +**Task Runner:** +- `poethepoet` - Unified task runner for development workflows + **General:** - `pre-commit` - Git hooks for automated quality checks - TOML formatting and validation -### Setting up pre-commit hooks +### Unified Development with Poe + +This project uses **Poe the Poet** for streamlined development: + +```bash +# Complete setup (installs deps, starts dev environment) +poe setup + +# Code quality (format, lint, type-check all code) +poe ci-quick + +# Run all tests +poe test-all + +# See all available tasks +poe --help +``` + +### Manual Setup (Alternative) ```bash pip install pre-commit pre-commit install ``` -### Running in Development Mode +### Quick Start -1. Clone the repository and navigate to the project directory: +1. **Clone the repository** + +2. **Unified Development (Recommended):** ```bash - cd plex-playlist + # Complete setup and start development environment + poe setup + + # Or manually: + cd backend + pip install -e . + poe dev-env-start ``` -2. Start the development environment: +3. **Traditional Setup:** ```bash - docker compose -f compose.yml -f compose.dev.yml up --build + # Backend development + cd backend + pip install -e . + + # Frontend development + cd frontend + npm install + npm run dev ``` -This will start: -- PostgreSQL database on port 5433 -- FastAPI backend on port 8001 (with hot reload) -- Vue.js frontend on port 5173 (with hot reload) +4. **Docker Development:** + ```bash + # Using Poe (recommended) + poe docker-dev-up + + # Or directly + docker compose -f compose.dev.yml up --build + ``` + +5. **Production Build:** + ```bash + poe docker-prod-up + # Or: docker compose up --build + ``` ### Running in Production Mode @@ -198,8 +244,14 @@ npm run dev ### Development & Workflow - **[Development Environment Setup](docs/DEVELOPMENT.md)** - Comprehensive guide for setting up your development environment, git workflow, pre-commit hooks, manual tool usage, and CI/CD pipeline understanding +- **[Poe Task Reference](docs/POE_TASK_REFERENCE.md)** - Complete guide to unified development tasks and workflows using Poe the Poet + +### Architecture & CI/CD + +- **[CI/CD Multi-stage Build](docs/CICD_MULTI_STAGE_BUILD.md)** - Docker multi-stage build strategy and optimization ### Operations & Troubleshooting + - **[Gitea Actions Troubleshooting](docs/GITEA_ACTIONS_TROUBLESHOOTING.md)** - Solutions for CI/CD pipeline issues, including the critical "jobs waiting forever" problem - **[Secure Docker CI/CD](docs/SECURE_DOCKER_CICD.md)** - Security considerations and setup for Docker-based CI/CD pipelines diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b3254ee..c6f8c2a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -19,7 +19,8 @@ dev = [ "yamllint>=1.35.0", "toml-sort>=0.23.0", "language-formatters-pre-commit-hooks>=2.14.0", # For pretty-format-toml - "xdoctest>=1.1.0" # For doctest support + "xdoctest>=1.1.0", # For doctest support + "poethepoet>=0.24.0" # Task runner for unified development workflows ] [project] @@ -82,6 +83,108 @@ packages = ["src/backend"] [tool.hatch.version] path = "src/backend/__init__.py" +[tool.poe.tasks] +build-cicd = {shell = "./scripts/build-cicd-local.sh", help = "Build both CI/CD images"} +# === Docker CI/CD Image Tasks === +build-cicd-base = {shell = "./scripts/build-cicd-local.sh --base-only", help = "Build CI/CD base image only"} +build-cicd-complete = {shell = "./scripts/build-cicd-local.sh --complete-only", help = "Build CI/CD complete image only"} +build-cicd-force = {shell = "./scripts/build-cicd-local.sh --force", help = "Force rebuild CI/CD images"} +build-if-dockerfile-changed = {shell = "if git diff --quiet HEAD~1 Dockerfile.cicd-base Dockerfile.cicd; then echo 'No Dockerfile changes, skipping build'; else poe build-cicd; fi", help = "Only build CI/CD images if Dockerfiles changed"} +# === Local CI Simulation === +ci-format-check = [ + {shell = "cd backend && uv run ruff format --check ."}, + {shell = "cd frontend && yarn prettier --check src/"} +] +ci-full = ["ci-format-check", "lint", "type-check", "test-all", "docs-backend"] +ci-quick = ["ci-format-check", "lint", "type-check"] +# === Utility Tasks === +clean = [ + {shell = "cd backend && find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true"}, + {shell = "cd backend && find . -name '*.pyc' -delete 2>/dev/null || true"}, + {shell = "cd frontend && rm -rf node_modules/.cache dist coverage 2>/dev/null || true"}, + {shell = "docker system prune -f"} +] +deps-check = [ + {shell = "cd backend && uv pip check"}, + {shell = "cd frontend && yarn audit"} +] +# === Dependency Management === +deps-install = [ + {shell = "cd backend && uv sync --dev"}, + {shell = "cd frontend && yarn install"} +] +deps-update = [ + {shell = "cd backend && uv sync --upgrade --dev"}, + {shell = "cd frontend && yarn upgrade"} +] +# === Development Environment Tasks === +dev = {shell = "docker compose -f compose.dev.yml up -d", help = "Start development environment"} +dev-down = {shell = "docker compose -f compose.dev.yml down", help = "Stop development environment"} +dev-logs = {shell = "docker compose -f compose.dev.yml logs -f", help = "Follow development environment logs"} +dev-restart = {shell = "docker compose -f compose.dev.yml restart", help = "Restart development environment"} +# === Documentation Tasks === +docs-backend = {shell = "cd backend && uv run xdoctest --module backend", help = "Run backend docstring tests"} +docs-check = {shell = "cd backend && uv run darglint backend/", help = "Check docstring quality"} +format = ["format-backend", "format-frontend"] +# === Code Formatting Tasks === +format-backend = {shell = "cd backend && uv run ruff format .", help = "Format backend Python code"} +format-frontend = {shell = "cd frontend && yarn prettier --write src/", help = "Format frontend TypeScript code"} +lint = ["lint-backend", "lint-frontend"] +# === Linting Tasks === +lint-backend = {shell = "cd backend && uv run ruff check .", help = "Lint backend Python code"} +lint-frontend = {shell = "cd frontend && yarn lint", help = "Lint frontend TypeScript code"} +# === Parallel Execution for Speed === +lint-parallel = {shell = "poe lint-backend & poe lint-frontend & wait", help = "Run linting in parallel for speed"} +# === CI/CD Tasks === +pre-commit-install = {shell = "pre-commit install", help = "Install pre-commit hooks"} +pre-commit-run = {shell = "pre-commit run --all-files", help = "Run all pre-commit hooks"} +pre-commit-update = {shell = "pre-commit autoupdate", help = "Update pre-commit hook versions"} +# === Quality Gates (Mimics CI Pipeline) === +quality-gate = [ + {shell = "echo '🔍 Running quality gate checks...'"}, + "ci-format-check", + "lint", + "type-check", + "test-unit", + "docs-check", + {shell = "echo '✅ All quality checks passed!'"} +] +reset = [ + "clean", + {shell = "cd backend && rm -rf .venv 2>/dev/null || true"}, + {shell = "cd frontend && rm -rf node_modules 2>/dev/null || true"}, + "deps-install" +] +# === Development Setup (New Developer Onboarding) === +setup = [ + "deps-install", + "pre-commit-install", + "dev", + {shell = "echo '✓ Development environment ready!'"}, + {shell = "echo ' - Frontend: http://localhost:3000'"}, + {shell = "echo ' - Backend API: http://localhost:8000'"}, + {shell = "echo ' - API Docs: http://localhost:8000/docs'"} +] +test-all = ["test-backend", "test-frontend", "test-integration"] +# === Testing Tasks === +test-backend = {shell = "cd backend && uv run pytest", help = "Run backend unit tests"} +test-backend-cov = {shell = "cd backend && uv run pytest --cov", help = "Run backend tests with coverage"} +test-e2e = {shell = "cd frontend && yarn test:e2e", help = "Run end-to-end tests"} +test-frontend = {shell = "cd frontend && yarn test", help = "Run frontend unit tests"} +test-frontend-cov = {shell = "cd frontend && yarn test:coverage", help = "Run frontend tests with coverage"} +test-full = ["test-backend-cov", "test-frontend-cov", "test-integration", "test-e2e"] +# === Smart Conditional Tasks === +test-if-changed = {shell = "if git diff --quiet HEAD~1 backend/ frontend/; then echo 'No changes detected, skipping tests'; else poe test-unit; fi", help = "Only run tests if code has changed"} +test-integration = {shell = "cd backend && uv run pytest tests/integration/", help = "Run backend integration tests"} +test-parallel = {shell = "poe test-backend & poe test-frontend & wait", help = "Run unit tests in parallel for speed"} +# === Comprehensive Testing === +test-unit = ["test-backend", "test-frontend"] +type-check = ["type-check-backend", "type-check-frontend"] +# === Type Checking Tasks === +type-check-backend = {shell = "cd backend && uv run pyright .", help = "Type check backend Python code"} +type-check-frontend = {shell = "cd frontend && yarn type-check", help = "Type check frontend TypeScript code"} +type-check-parallel = {shell = "poe type-check-backend & poe type-check-frontend & wait", help = "Run type checking in parallel for speed"} + [tool.pytest.ini_options] addopts = [ "--strict-markers", diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index a58c1ee..3324a58 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -6,14 +6,16 @@ This document outlines how to set up your development environment and work with 1. [Quick Start](#quick-start) 2. [Development Environment Setup](#development-environment-setup) -3. [Git Workflow](#git-workflow) -4. [Pre-commit Hooks](#pre-commit-hooks) -5. [Manual Tool Usage](#manual-tool-usage) -6. [CI/CD Pipeline](#cicd-pipeline) -7. [Branch Protection and Merge Requirements](#branch-protection-and-merge-requirements) +3. [Poe the Poet Task Runner](#poe-the-poet-task-runner) +4. [Git Workflow](#git-workflow) +5. [Pre-commit Hooks](#pre-commit-hooks) +6. [Manual Tool Usage](#manual-tool-usage) +7. [CI/CD Pipeline](#cicd-pipeline) +8. [Branch Protection and Merge Requirements](#branch-protection-and-merge-requirements) ## Related Documentation +- **[Poe Task Reference](POE_TASK_REFERENCE.md)** - Complete guide to unified development tasks - **[CI/CD Multi-Stage Build Architecture](CICD_MULTI_STAGE_BUILD.md)** - Technical details of the optimized build system - **[CI/CD Troubleshooting](GITEA_ACTIONS_TROUBLESHOOTING.md)** - Common issues and solutions - **[Secure Docker CI/CD](SECURE_DOCKER_CICD.md)** - Security considerations and practices @@ -25,15 +27,19 @@ This document outlines how to set up your development environment and work with git clone ssh://git@dogar.darkhelm.org:2222/DarkHelm.org/plex-playlist.git cd plex-playlist -# Start development environment -docker compose -f compose.dev.yml up -d +# Complete setup with Poe the Poet (recommended) +cd backend && uv sync --dev && cd .. +poe setup -# Set up pre-commit (recommended) -pip install pre-commit -pre-commit install +# OR manual setup +docker compose -f compose.dev.yml up -d +pip install pre-commit && pre-commit install # Create a feature branch and start developing git checkout -b feature/your-feature-name + +# Use Poe for all development tasks +poe format lint type-check test-unit ``` ## Development Environment Setup @@ -68,6 +74,55 @@ docker compose -f compose.dev.yml up -d --build - **API Documentation**: http://localhost:8000/docs - **Database**: localhost:5432 (user: `plex`, password: see `secrets/postgres_password`) +## Poe the Poet Task Runner + +This project uses **Poe the Poet** as a unified task runner to simplify and standardize development workflows. Instead of remembering different commands for backend and frontend, you can use consistent `poe` commands from anywhere in the project. + +### Quick Start with Poe + +```bash +# Complete development setup (new developers) +poe setup + +# Start development environment +poe dev + +# Run all quality checks (format, lint, type-check) +poe ci-quick + +# Run all tests +poe test-all + +# See all available tasks +poe --help +``` + +### Key Benefits + +- **Unified Interface**: Same commands work for backend (Python) and frontend (TypeScript) +- **Parallel Execution**: Tasks run in parallel for better performance +- **Smart Workflows**: Conditional and chained task execution +- **Developer Friendly**: Interactive task selection and helpful descriptions +- **CI Integration**: Local simulation of CI/CD pipeline + +### Common Workflows + +```bash +# Daily development workflow +poe format lint type-check test-unit + +# Pre-commit workflow (faster than full CI) +poe ci-quick + +# Complete quality gate (like CI pipeline) +poe quality-gate + +# Build and test Docker images locally +poe build-cicd +``` + +For the complete list of tasks and detailed usage, see **[Poe Task Reference](POE_TASK_REFERENCE.md)**. + ## Git Workflow ### Branch Strategy diff --git a/docs/POE_TASK_REFERENCE.md b/docs/POE_TASK_REFERENCE.md new file mode 100644 index 0000000..c7f64ef --- /dev/null +++ b/docs/POE_TASK_REFERENCE.md @@ -0,0 +1,178 @@ +# Poe the Poet Task Reference + +This project uses **Poe the Poet** as a unified task runner to simplify development workflows. All tasks are defined in `backend/pyproject.toml` and can be run from the project root. + +## 🚀 Quick Start + +```bash +# Install dependencies and set up development environment +poe setup + +# See all available tasks +poe --help + +# Interactive task selection +poe +``` + +## 📋 Essential Tasks + +### Development Environment +```bash +poe dev # Start development environment (Docker Compose) +poe dev-down # Stop development environment +poe dev-logs # Follow development logs +poe dev-restart # Restart development environment +``` + +### Code Quality (Unified Backend + Frontend) +```bash +poe format # Format all code (Python + TypeScript) +poe lint # Lint all code (Python + TypeScript) +poe type-check # Type check all code (Python + TypeScript) +``` + +### Testing +```bash +poe test-unit # Run all unit tests (backend + frontend) +poe test-all # Run all tests including integration +poe test-full # Run all tests with coverage + E2E +poe test-e2e # Run end-to-end tests only +``` + +### CI/CD Pipeline +```bash +poe ci-quick # Fast quality checks (format, lint, type-check) +poe ci-full # Complete CI pipeline simulation +poe quality-gate # All quality checks (like CI) +``` + +### Docker Images +```bash +poe build-cicd # Build both base and complete CI/CD images +poe build-cicd-base # Build only base image (cached dependencies) +poe build-cicd-complete # Build only complete image (fast) +``` + +## 🏃‍♂️ Performance Tasks (Parallel Execution) + +```bash +poe lint-parallel # Run linting in parallel for speed +poe type-check-parallel # Run type checking in parallel +poe test-parallel # Run tests in parallel +``` + +## 🧠 Smart Tasks (Conditional Execution) + +```bash +poe test-if-changed # Only test if code changed +poe build-if-dockerfile-changed # Only build if Dockerfiles changed +``` + +## 🛠 Utility Tasks + +```bash +poe deps-install # Install all dependencies (backend + frontend) +poe deps-update # Update all dependencies +poe clean # Clean build artifacts and caches +poe reset # Complete reset (clean + reinstall) +``` + +## 🔧 Individual Component Tasks + +### Backend Only +```bash +poe format-backend # Format Python code only +poe lint-backend # Lint Python code only +poe type-check-backend # Type check Python only +poe test-backend # Run backend tests only +poe test-backend-cov # Backend tests with coverage +``` + +### Frontend Only +```bash +poe format-frontend # Format TypeScript code only +poe lint-frontend # Lint TypeScript code only +poe type-check-frontend # Type check TypeScript only +poe test-frontend # Run frontend tests only +poe test-frontend-cov # Frontend tests with coverage +``` + +## 📚 Documentation Tasks + +```bash +poe docs-backend # Run docstring tests +poe docs-check # Check docstring quality +``` + +## 🎣 Pre-commit Integration + +```bash +poe pre-commit-install # Install pre-commit hooks +poe pre-commit-run # Run all pre-commit hooks +poe pre-commit-update # Update hook versions +``` + +## 💡 Tips & Tricks + +### Task Discovery +```bash +poe --help # List all tasks with descriptions +poe --help # Get help for specific task +poe # Interactive task picker +``` + +### Chaining Tasks +```bash +# Run multiple tasks in sequence +poe format lint type-check test-unit + +# Custom workflows +poe clean deps-install ci-quick +``` + +### Environment Context +- All tasks run from project root +- Backend tasks automatically use `uv run` in correct environment +- Frontend tasks automatically use `yarn` in correct directory +- Docker tasks use the optimized multi-stage CI/CD setup + +### Performance Tips +- Use parallel tasks (`*-parallel`) for faster feedback +- Use conditional tasks (`*-if-changed`) to save time +- `poe setup` configures everything for new developers +- `poe quality-gate` runs full CI checks locally + +## 🔄 Migration from Manual Commands + +### Before (Manual) +```bash +cd backend && uv run ruff format . +cd frontend && yarn prettier --write src/ +cd backend && uv run pytest +cd frontend && yarn test +./scripts/build-cicd-local.sh +``` + +### After (Poe) +```bash +poe format +poe test-unit +poe build-cicd +``` + +## 🏗 Integration with Existing Workflows + +- **Pre-commit hooks**: Still work as before, Poe complements them +- **CI/CD pipeline**: Uses same tools, Poe provides local simulation +- **Docker development**: `poe dev` replaces manual docker-compose commands +- **Scripts**: Existing scripts still work, Poe provides unified interface + +## 🎯 Next Steps + +1. **Try it out**: `poe setup` to get started +2. **Explore tasks**: `poe --help` to see all options +3. **Customize**: Add project-specific tasks to `backend/pyproject.toml` +4. **Share**: Team members can use same unified interface + +This unified task system makes development faster, more consistent, and easier for new team members to learn!