From 1f3785a7a3c22350d6880ce14d8bf05c3059a7c6 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 31 Jan 2022 18:35:39 +0000 Subject: [PATCH 1/2] add script to run celery from within docker as a team we primarily develop locally. However, we've been experiencing issues with pycurl, a subdependency of celery, that is notoriously difficult to install on mac. On top of the existing issues, we're also seeing it conflict with pyproj in bizarre ways (where the order of imports between pyproj and pycurl result in different configurations of dynamically linked C libraries being loaded. You are encouraged to attempt to install pycurl locally, following these instructions: https://github.com/alphagov/notifications-manuals/wiki/Getting-Started#pycurl However, if you aren't having any luck, you can instead now run celery in a docker container. `make run-celery-with-docker` This will build a container, install the dependencies, and run celery (with the default of four concurrent workers). It will pull aws variables from your aws configuration as boto would normally, and it will attempt to connect to your local database with the user `postgres`. If your local database is configured differently (for example, with a different user, or on a different port), then you can set the SQLALCHEMY_DATABASE_URI locally to override that. --- Makefile | 5 +++++ app/config.py | 2 +- docker/Dockerfile | 32 ++++++++++++++++++++++++++++++++ scripts/run_with_docker.sh | 13 +++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile create mode 100755 scripts/run_with_docker.sh diff --git a/Makefile b/Makefile index 0204d01d3..1fc5d9f24 100644 --- a/Makefile +++ b/Makefile @@ -177,3 +177,8 @@ disable-failwhale: ## Disable the failwhale app and enable api cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api cf stop notify-api-failwhale @echo "Failwhale is disabled" + +.PHONY: run-celery-with-docker +run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally) + docker build -f docker/Dockerfile -t notifications-api . + ./scripts/run_with_docker.sh make run-celery diff --git a/app/config.py b/app/config.py index 0afc852ed..b4df84345 100644 --- a/app/config.py +++ b/app/config.py @@ -427,7 +427,7 @@ class Development(Config): NOTIFY_LOG_PATH = 'application.log' NOTIFY_EMAIL_DOMAIN = "notify.tools" - SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api' + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api') REDIS_URL = 'redis://localhost:6379/0' ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..d083a6f64 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.9-slim-bullseye as parent + +ENV PYTHONUNBUFFERED=1 +ENV DEBIAN_FRONTEND=noninteractive + +RUN echo "Install base packages" && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + && echo "Install binary app dependencies" \ + && apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev \ + libssl-dev \ + && apt-get -y clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* + +RUN pip install --upgrade pip + +WORKDIR /home/vcap/app + +COPY requirements.txt ./ + +# RUN useradd celeryuser + +RUN \ + echo "Installing python dependencies" \ + && pip install -r requirements.txt + +COPY app app +COPY run_celery.py . +COPY environment.sh . +COPY Makefile . diff --git a/scripts/run_with_docker.sh b/scripts/run_with_docker.sh new file mode 100755 index 000000000..e97adbfae --- /dev/null +++ b/scripts/run_with_docker.sh @@ -0,0 +1,13 @@ +#!/bin/bash +DOCKER_IMAGE_NAME=notifications-api + +source environment.sh + +docker run -it --rm \ + -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-$(aws configure get aws_access_key_id)} \ + -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-$(aws configure get aws_secret_access_key)} \ + -e SQLALCHEMY_DATABASE_URI=${SQLALCHEMY_DATABASE_URI:-postgresql://postgres@host.docker.internal/notification_api} \ + -v $(pwd):/home/vcap/app \ + ${DOCKER_ARGS} \ + ${DOCKER_IMAGE_NAME} \ + ${@} From 39848e6df0635c1cdd837c25fa0d055bd22e54f2 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 1 Feb 2022 14:19:52 +0000 Subject: [PATCH 2/2] move environment variables to their own lines and set -eu this means that if the environment variable can't be set (for example, if you don't have aws-cli installed) then there's a suitable error message early on. --- scripts/run_with_docker.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/run_with_docker.sh b/scripts/run_with_docker.sh index e97adbfae..b423a7923 100755 --- a/scripts/run_with_docker.sh +++ b/scripts/run_with_docker.sh @@ -1,13 +1,19 @@ #!/bin/bash +set -eu + DOCKER_IMAGE_NAME=notifications-api source environment.sh +# this script should be run from within your virtualenv so you can access the aws cli +AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"$(aws configure get aws_access_key_id)"} +AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"$(aws configure get aws_secret_access_key)"} +: "${SQLALCHEMY_DATABASE_URI:=postgresql://postgres@host.docker.internal/notification_api}" + docker run -it --rm \ - -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-$(aws configure get aws_access_key_id)} \ - -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-$(aws configure get aws_secret_access_key)} \ - -e SQLALCHEMY_DATABASE_URI=${SQLALCHEMY_DATABASE_URI:-postgresql://postgres@host.docker.internal/notification_api} \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e SQLALCHEMY_DATABASE_URI=$SQLALCHEMY_DATABASE_URI \ -v $(pwd):/home/vcap/app \ - ${DOCKER_ARGS} \ ${DOCKER_IMAGE_NAME} \ ${@}