From c9a9640a4b24b3138f9431033e17beb602f0fd12 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Thu, 24 Feb 2022 17:11:46 +0000 Subject: [PATCH 1/3] Iterate local development with Docker This makes a few changes to: - Make local development consistent with our other apps. It's now faster to start Celery locally since we don't try to build the image each time - this is usually quick, but unnecessary. - Add support for connecting to a local Redis instance. Note that the previous suggestion of "REDIS = True" was incorrect as this would be turned into the literal string "True". I've also co-located and extended the recipes in the Makefile to make them a bit more visible. --- Makefile | 17 ++++++++++++----- README.md | 17 ++++++++++++++++- app/config.py | 2 +- scripts/run_with_docker.sh | 2 ++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 1fc5d9f24..2a8a6ac7c 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,10 @@ bootstrap: generate-version-file ## Set up everything to run the app createdb notification_api || true (. environment.sh && flask db upgrade) || true +.PHONY: bootstrap-with-docker +bootstrap-with-docker: ## Build the image to run the app in Docker + docker build -f docker/Dockerfile -t notifications-api . + .PHONY: run-flask run-flask: ## Run flask . environment.sh && flask run -p 6011 @@ -39,12 +43,20 @@ run-celery: ## Run celery --loglevel=INFO \ --concurrency=4 +.PHONY: run-celery-with-docker +run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally) + ./scripts/run_with_docker.sh make run-celery + .PHONY: run-celery-beat run-celery-beat: ## Run celery beat . environment.sh && celery \ -A run_celery.notify_celery beat \ --loglevel=INFO +.PHONY: run-celery-beat-with-docker +run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally) + ./scripts/run_with_docker.sh make run-celery-beat + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -177,8 +189,3 @@ disable-failwhale: ## Disable the failwhale app and enable api cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api cf stop notify-api-failwhale @echo "Failwhale is disabled" - -.PHONY: run-celery-with-docker -run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally) - docker build -f docker/Dockerfile -t notifications-api . - ./scripts/run_with_docker.sh make run-celery diff --git a/README.md b/README.md index 223870d3e..4ae66cec4 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,11 @@ export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development: - REDIS_ENABLED = True +``` +REDIS_ENABLED = 1 +``` +Note that if you're running Celery with Docker locally, then you'll also need to set `REDIS_URL=redis://host.docker.internal:6379`. ## To run the application @@ -82,6 +85,18 @@ make run-celery make run-celery-beat ``` +We've had problems running Celery locally due to one of its dependencies: pycurl. Due to the complexity of the issue, we also support running Celery via Docker: + +``` +# install dependencies, etc. +make bootstrap-with-docker + +# run the background tasks +make run-celery-with-docker + +# run scheduled tasks +make run-celery-beat-with-docker + ## To test the application ``` diff --git a/app/config.py b/app/config.py index 5277945c3..37b0f4619 100644 --- a/app/config.py +++ b/app/config.py @@ -428,7 +428,7 @@ class Development(Config): NOTIFY_EMAIL_DOMAIN = "notify.tools" SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api') - REDIS_URL = 'redis://localhost:6379/0' + REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' diff --git a/scripts/run_with_docker.sh b/scripts/run_with_docker.sh index b423a7923..b53d729e4 100755 --- a/scripts/run_with_docker.sh +++ b/scripts/run_with_docker.sh @@ -14,6 +14,8 @@ docker run -it --rm \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e SQLALCHEMY_DATABASE_URI=$SQLALCHEMY_DATABASE_URI \ + -e REDIS_ENABLED=${REDIS_ENABLED:-0} \ + -e REDIS_URL=${REDIS_URL:-''} \ -v $(pwd):/home/vcap/app \ ${DOCKER_IMAGE_NAME} \ ${@} From 038d47e7027ada57791ae44e99a4428607191d5a Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Fri, 25 Feb 2022 17:49:07 +0000 Subject: [PATCH 2/3] Minor tweaks in response to PR comments --- README.md | 6 ++---- scripts/run_with_docker.sh | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4ae66cec4..da5e9e33f 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,12 @@ export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ ### Redis -To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development: +To switch redis on you'll need to install it locally e.g. `brew install redis`. To use redis caching you need to switch it on with an environment variable: ``` -REDIS_ENABLED = 1 +export REDIS_ENABLED=1 ``` -Note that if you're running Celery with Docker locally, then you'll also need to set `REDIS_URL=redis://host.docker.internal:6379`. - ## To run the application ``` diff --git a/scripts/run_with_docker.sh b/scripts/run_with_docker.sh index b53d729e4..70933d111 100755 --- a/scripts/run_with_docker.sh +++ b/scripts/run_with_docker.sh @@ -9,13 +9,14 @@ source environment.sh AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"$(aws configure get aws_access_key_id)"} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"$(aws configure get aws_secret_access_key)"} : "${SQLALCHEMY_DATABASE_URI:=postgresql://postgres@host.docker.internal/notification_api}" +REDIS_URL="redis://host.docker.internal:6379" docker run -it --rm \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e SQLALCHEMY_DATABASE_URI=$SQLALCHEMY_DATABASE_URI \ -e REDIS_ENABLED=${REDIS_ENABLED:-0} \ - -e REDIS_URL=${REDIS_URL:-''} \ + -e REDIS_URL=$REDIS_URL \ -v $(pwd):/home/vcap/app \ ${DOCKER_IMAGE_NAME} \ ${@} From bd690ab71825b227b01d9fc3aa347dff146e1a5a Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Mon, 28 Feb 2022 11:27:34 +0000 Subject: [PATCH 3/3] Be more helpful for how to install / start Redis In response to [1]. [1]: https://github.com/alphagov/notifications-api/pull/3467#discussion_r815797015 --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da5e9e33f..479277228 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,15 @@ export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ ### Redis -To switch redis on you'll need to install it locally e.g. `brew install redis`. To use redis caching you need to switch it on with an environment variable: +To switch redis on you'll need to install it locally. On a Mac you can do: + +``` +# assuming you use Homebrew +brew install redis +brew services start redis +``` + +To use redis caching you need to switch it on with an environment variable: ``` export REDIS_ENABLED=1